|
|
reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
|
Declarations
include/llvm/CodeGen/Analysis.h 35 class SDValue;
include/llvm/CodeGen/SelectionDAGISel.h 28 class SDValue;
References
gen/lib/Target/AArch64/AArch64GenDAGISel.inc115484 SDValue N, unsigned PatternNo,
115485 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
115630 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
115630 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/AMDGPU/AMDGPUGenDAGISel.inc78718 SDValue Src0 = N->getOperand(0);
78719 SDValue Src1 = N->getOperand(1);
79609 bool DAGISEL_CLASS_COLONCOLON CheckNodePredicateWithOperands(SDNode *Node, unsigned PredNo, const SmallVectorImpl<SDValue> &Operands) const
79662 SDValue N, unsigned PatternNo,
79663 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
79778 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
79778 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/AMDGPU/R600GenDAGISel.inc12583 SDValue N, unsigned PatternNo,
12584 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
12612 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
12612 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/ARC/ARCGenDAGISel.inc 1219 SDValue N, unsigned PatternNo,
1220 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
gen/lib/Target/ARM/ARMGenDAGISel.inc54869 SDValue N, unsigned PatternNo,
54870 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
55015 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
55015 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/AVR/AVRGenDAGISel.inc 1724 SDValue N, unsigned PatternNo,
1725 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
1744 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
1744 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/BPF/BPFGenDAGISel.inc 2104 SDValue N, unsigned PatternNo,
2105 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
gen/lib/Target/Hexagon/HexagonGenDAGISel.inc72650 SDValue Op1 = N->getOperand(1);
72896 SDValue N, unsigned PatternNo,
72897 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result) override;
72901 SDValue N, unsigned PatternNo,
72902 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
72942 SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo) override;
72942 SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo) override;
72945 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
72945 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/Lanai/LanaiGenDAGISel.inc 1512 SDValue N, unsigned PatternNo,
1513 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
1541 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
1541 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/MSP430/MSP430GenDAGISel.inc 4885 SDValue N, unsigned PatternNo,
4886 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
gen/lib/Target/Mips/MipsGenDAGISel.inc30479 SDValue N, unsigned PatternNo,
30480 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
30613 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
30613 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/NVPTX/NVPTXGenDAGISel.inc72319 SDValue N, unsigned PatternNo,
72320 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
72345 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
72345 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/PowerPC/PPCGenDAGISel.inc44827 SDValue N, unsigned PatternNo,
44828 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
44871 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
44871 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/RISCV/RISCVGenDAGISel.inc14159 SDValue N, unsigned PatternNo,
14160 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
14179 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
14179 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/Sparc/SparcGenDAGISel.inc 3602 SDValue N, unsigned PatternNo,
3603 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
3625 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
3625 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/SystemZ/SystemZGenDAGISel.inc30435 SDValue N, unsigned PatternNo,
30436 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
30503 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
30503 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/X86/X86GenDAGISel.inc254667 SDValue N, unsigned PatternNo,
254668 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
254723 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
254723 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
gen/lib/Target/XCore/XCoreGenDAGISel.inc 2446 SDValue N, unsigned PatternNo,
2447 SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)
2466 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
2466 SDValue DAGISEL_CLASS_COLONCOLON RunSDNodeXForm(SDValue V, unsigned XFormNo)
include/llvm/ADT/ArrayRef.h 43 using iterator = const T *;
44 using const_iterator = const T *;
50 const T *Data = nullptr;
66 /*implicit*/ ArrayRef(const T &OneElt)
70 /*implicit*/ ArrayRef(const T *data, size_t length)
74 ArrayRef(const T *begin, const T *end)
74 ArrayRef(const T *begin, const T *end)
81 /*implicit*/ ArrayRef(const SmallVectorTemplateCommon<T, U> &Vec)
87 /*implicit*/ ArrayRef(const std::vector<T, A> &Vec)
92 /*implicit*/ constexpr ArrayRef(const std::array<T, N> &Arr)
97 /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {}
100 /*implicit*/ ArrayRef(const std::initializer_list<T> &Vec)
145 const T *data() const { return Data; }
151 const T &front() const {
157 const T &back() const {
163 template <typename Allocator> ArrayRef<T> copy(Allocator &A) {
178 ArrayRef<T> slice(size_t N, size_t M) const {
184 ArrayRef<T> slice(size_t N) const { return slice(N, size() - N); }
187 ArrayRef<T> drop_front(size_t N = 1) const {
193 ArrayRef<T> drop_back(size_t N = 1) const {
200 template <class PredicateT> ArrayRef<T> drop_while(PredicateT Pred) const {
206 template <class PredicateT> ArrayRef<T> drop_until(PredicateT Pred) const {
211 ArrayRef<T> take_front(size_t N = 1) const {
218 ArrayRef<T> take_back(size_t N = 1) const {
226 template <class PredicateT> ArrayRef<T> take_while(PredicateT Pred) const {
232 template <class PredicateT> ArrayRef<T> take_until(PredicateT Pred) const {
239 const T &operator[](size_t Index) const {
249 typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type &
257 typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type &
263 std::vector<T> vec() const {
270 operator std::vector<T>() const {
290 class LLVM_NODISCARD MutableArrayRef : public ArrayRef<T> {
292 using iterator = T *;
302 /*implicit*/ MutableArrayRef(T &OneElt) : ArrayRef<T>(OneElt) {}
305 /*implicit*/ MutableArrayRef(T *data, size_t length)
309 MutableArrayRef(T *begin, T *end) : ArrayRef<T>(begin, end) {}
309 MutableArrayRef(T *begin, T *end) : ArrayRef<T>(begin, end) {}
312 /*implicit*/ MutableArrayRef(SmallVectorImpl<T> &Vec)
316 /*implicit*/ MutableArrayRef(std::vector<T> &Vec)
321 /*implicit*/ constexpr MutableArrayRef(std::array<T, N> &Arr)
326 /*implicit*/ constexpr MutableArrayRef(T (&Arr)[N]) : ArrayRef<T>(Arr) {}
328 T *data() const { return const_cast<T*>(ArrayRef<T>::data()); }
337 T &front() const {
343 T &back() const {
350 MutableArrayRef<T> slice(size_t N, size_t M) const {
356 MutableArrayRef<T> slice(size_t N) const {
361 MutableArrayRef<T> drop_front(size_t N = 1) const {
366 MutableArrayRef<T> drop_back(size_t N = 1) const {
374 MutableArrayRef<T> drop_while(PredicateT Pred) const {
381 MutableArrayRef<T> drop_until(PredicateT Pred) const {
386 MutableArrayRef<T> take_front(size_t N = 1) const {
393 MutableArrayRef<T> take_back(size_t N = 1) const {
402 MutableArrayRef<T> take_while(PredicateT Pred) const {
409 MutableArrayRef<T> take_until(PredicateT Pred) const {
416 T &operator[](size_t Index) const {
456 ArrayRef<T> makeArrayRef(const T *data, size_t length) {
456 ArrayRef<T> makeArrayRef(const T *data, size_t length) {
468 ArrayRef<T> makeArrayRef(const SmallVectorImpl<T> &Vec) {
468 ArrayRef<T> makeArrayRef(const SmallVectorImpl<T> &Vec) {
474 ArrayRef<T> makeArrayRef(const SmallVector<T, N> &Vec) {
474 ArrayRef<T> makeArrayRef(const SmallVector<T, N> &Vec) {
include/llvm/ADT/DenseMap.h 40 struct DenseMapPair : public std::pair<KeyT, ValueT> {
40 struct DenseMapPair : public std::pair<KeyT, ValueT> {
43 KeyT &getFirst() { return std::pair<KeyT, ValueT>::first; }
44 const KeyT &getFirst() const { return std::pair<KeyT, ValueT>::first; }
45 ValueT &getSecond() { return std::pair<KeyT, ValueT>::second; }
46 const ValueT &getSecond() const { return std::pair<KeyT, ValueT>::second; }
65 using key_type = KeyT;
66 using mapped_type = ValueT;
69 using iterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT>;
69 using iterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT>;
71 DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>;
71 DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>;
78 if (shouldReverseIterate<KeyT>())
121 const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
122 if (is_trivially_copyable<KeyT>::value &&
123 is_trivially_copyable<ValueT>::value) {
145 size_type count(const_arg_type_t<KeyT> Val) const {
150 iterator find(const_arg_type_t<KeyT> Val) {
156 const_iterator find(const_arg_type_t<KeyT> Val) const {
185 ValueT lookup(const_arg_type_t<KeyT> Val) const {
185 ValueT lookup(const_arg_type_t<KeyT> Val) const {
195 std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) {
195 std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) {
202 std::pair<iterator, bool> insert(std::pair<KeyT, ValueT> &&KV) {
202 std::pair<iterator, bool> insert(std::pair<KeyT, ValueT> &&KV) {
210 std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&... Args) {
219 InsertIntoBucket(TheBucket, std::move(Key), std::forward<Ts>(Args)...);
229 std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&... Args) {
249 std::pair<iterator, bool> insert_as(std::pair<KeyT, ValueT> &&KV,
249 std::pair<iterator, bool> insert_as(std::pair<KeyT, ValueT> &&KV,
272 bool erase(const KeyT &Val) {
291 value_type& FindAndConstruct(const KeyT &Key) {
299 ValueT &operator[](const KeyT &Key) {
299 ValueT &operator[](const KeyT &Key) {
303 value_type& FindAndConstruct(KeyT &&Key) {
311 ValueT &operator[](KeyT &&Key) {
311 ValueT &operator[](KeyT &&Key) {
334 const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
349 const KeyT EmptyKey = getEmptyKey();
351 ::new (&B->getFirst()) KeyT(EmptyKey);
369 const KeyT EmptyKey = getEmptyKey();
370 const KeyT TombstoneKey = getTombstoneKey();
380 ::new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond()));
392 const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) {
392 const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) {
414 static unsigned getHashValue(const KeyT &Val) {
419 static unsigned getHashValue(const LookupKeyT &Val) {
423 static const KeyT getEmptyKey() {
429 static const KeyT getTombstoneKey() {
437 if (shouldReverseIterate<KeyT>()) {
515 BucketT *InsertIntoBucket(BucketT *TheBucket, KeyArg &&Key,
516 ValueArgs &&... Values) {
519 TheBucket->getFirst() = std::forward<KeyArg>(Key);
520 ::new (&TheBucket->getSecond()) ValueT(std::forward<ValueArgs>(Values)...);
520 ::new (&TheBucket->getSecond()) ValueT(std::forward<ValueArgs>(Values)...);
525 BucketT *InsertIntoBucketWithLookup(BucketT *TheBucket, KeyT &&Key,
526 ValueT &&Value, LookupKeyT &Lookup) {
535 BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup,
535 BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup,
566 const KeyT EmptyKey = getEmptyKey();
578 bool LookupBucketFor(const LookupKeyT &Val,
590 const KeyT EmptyKey = getEmptyKey();
591 const KeyT TombstoneKey = getTombstoneKey();
629 bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
684 class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
684 class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
685 KeyT, ValueT, KeyInfoT, BucketT> {
685 KeyT, ValueT, KeyInfoT, BucketT> {
690 using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
690 using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
852 SmallDenseMap<KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT>, KeyT,
852 SmallDenseMap<KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT>, KeyT,
852 SmallDenseMap<KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT>, KeyT,
853 ValueT, KeyInfoT, BucketT> {
1023 const KeyT EmptyKey = this->getEmptyKey();
1024 const KeyT TombstoneKey = this->getTombstoneKey();
1030 ::new (&TmpEnd->getFirst()) KeyT(std::move(P->getFirst()));
1031 ::new (&TmpEnd->getSecond()) ValueT(std::move(P->getSecond()));
1163 using ConstIterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>;
1163 using ConstIterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>;
1186 if (shouldReverseIterate<KeyT>()) {
1199 const DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, IsConstSrc> &I)
1199 const DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, IsConstSrc> &I)
1210 if (shouldReverseIterate<KeyT>())
1232 if (shouldReverseIterate<KeyT>()) {
1249 const KeyT Empty = KeyInfoT::getEmptyKey();
1250 const KeyT Tombstone = KeyInfoT::getTombstoneKey();
1259 const KeyT Empty = KeyInfoT::getEmptyKey();
1260 const KeyT Tombstone = KeyInfoT::getTombstoneKey();
include/llvm/ADT/DenseMapInfo.h 186 using Pair = std::pair<T, U>;
187 using FirstInfo = DenseMapInfo<T>;
include/llvm/ADT/STLExtras.h 1023 constexpr inline size_t array_lengthof(T (&)[N]) {
1224 bool is_contained(R &&Range, const E &Element) {
include/llvm/ADT/SmallSet.h 138 SmallVector<T, N> Vector;
139 std::set<T, C> Set;
141 using VIterator = typename SmallVector<T, N>::const_iterator;
142 using mutable_iterator = typename SmallVector<T, N>::iterator;
151 using const_iterator = SmallSetIterator<T, N, C>;
164 size_type count(const T &V) const {
180 std::pair<NoneType, bool> insert(const T &V) {
207 bool erase(const T &V) {
238 VIterator vfind(const T &V) const {
include/llvm/ADT/SmallVector.h 75 AlignedCharArrayUnion<T> FirstEl;
114 using value_type = T;
115 using iterator = T *;
116 using const_iterator = const T *;
121 using reference = T &;
122 using const_reference = const T &;
123 using pointer = T *;
124 using const_pointer = const T *;
259 class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> {
264 static void destroy_range(T *, T *) {}
264 static void destroy_range(T *, T *) {}
286 T1 *I, T1 *E, T2 *Dest,
286 T1 *I, T1 *E, T2 *Dest,
286 T1 *I, T1 *E, T2 *Dest,
287 typename std::enable_if<std::is_same<typename std::remove_const<T1>::type,
288 T2>::value>::type * = nullptr) {
294 memcpy(reinterpret_cast<void *>(Dest), I, (E - I) * sizeof(T));
299 void grow(size_t MinSize = 0) { this->grow_pod(MinSize, sizeof(T)); }
302 void push_back(const T &Elt) {
305 memcpy(reinterpret_cast<void *>(this->end()), &Elt, sizeof(T));
315 class SmallVectorImpl : public SmallVectorTemplateBase<T> {
316 using SuperClass = SmallVectorTemplateBase<T>;
352 new (&*I) T();
357 void resize(size_type N, const T &NV) {
374 LLVM_NODISCARD T pop_back_val() {
375 T Result = ::std::move(this->back());
397 void append(size_type NumInputs, const T &Elt) {
405 void append(std::initializer_list<T> IL) {
412 void assign(size_type NumElts, const T &Elt) {
429 void assign(std::initializer_list<T> IL) {
467 iterator insert(iterator I, T &&Elt) {
482 ::new ((void*) this->end()) T(::std::move(this->back()));
489 T *EltPtr = &Elt;
497 iterator insert(iterator I, const T &Elt) {
511 ::new ((void*) this->end()) T(std::move(this->back()));
518 const T *EltPtr = &Elt;
526 iterator insert(iterator I, size_type NumToInsert, const T &Elt) {
606 T *OldEnd = this->end();
621 T *OldEnd = this->end();
627 for (T *J = I; NumOverwritten > 0; --NumOverwritten) {
637 void insert(iterator I, std::initializer_list<T> IL) {
641 template <typename... ArgTypes> reference emplace_back(ArgTypes &&... Args) {
644 ::new ((void *)this->end()) T(std::forward<ArgTypes>(Args)...);
820 AlignedCharArrayUnion<T> InlineElts[N];
837 class SmallVector : public SmallVectorImpl<T>, SmallVectorStorage<T, N> {
837 class SmallVector : public SmallVectorImpl<T>, SmallVectorStorage<T, N> {
846 explicit SmallVector(size_t Size, const T &Value = T())
865 SmallVector(std::initializer_list<T> IL) : SmallVectorImpl<T>(N) {
884 SmallVector(SmallVectorImpl<T> &&RHS) : SmallVectorImpl<T>(N) {
include/llvm/ADT/iterator.h 68 : public std::iterator<IteratorCategoryT, T, DifferenceTypeT, PointerT,
206 : public iterator_facade_base<DerivedT, IteratorCategoryT, T,
include/llvm/CodeGen/SelectionDAG.h 242 SDValue Root;
379 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals);
474 const SDValue &getRoot() const { return Root; }
477 SDValue getEntryNode() const {
483 const SDValue &setRoot(SDValue N) {
483 const SDValue &setRoot(SDValue N) {
578 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
580 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
583 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
589 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
591 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL,
593 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL,
596 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT,
600 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT,
604 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
611 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT);
622 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT,
624 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT,
626 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT,
628 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) {
631 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) {
634 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) {
639 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
642 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
646 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false);
647 SDValue getTargetFrameIndex(int FI, EVT VT) {
650 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false,
652 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) {
655 SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align = 0,
658 SDValue getTargetConstantPool(const Constant *C, EVT VT, unsigned Align = 0,
662 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT,
665 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT,
670 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
674 SDValue getBasicBlock(MachineBasicBlock *MBB);
675 SDValue getBasicBlock(MachineBasicBlock *MBB, SDLoc dl);
676 SDValue getExternalSymbol(const char *Sym, EVT VT);
677 SDValue getExternalSymbol(const char *Sym, const SDLoc &dl, EVT VT);
678 SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
680 SDValue getMCSymbol(MCSymbol *Sym, EVT VT);
682 SDValue getValueType(EVT);
683 SDValue getRegister(unsigned Reg, EVT VT);
684 SDValue getRegisterMask(const uint32_t *RegMask);
685 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
685 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
686 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
686 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
688 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0,
690 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT,
695 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg,
695 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg,
696 SDValue N) {
704 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
704 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
704 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
705 SDValue Glue) {
707 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
713 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
713 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
713 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
713 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
714 SDValue Glue) {
716 SDValue Ops[] = { Chain, Reg, N, Glue };
721 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) {
721 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) {
723 SDValue Ops[] = { Chain, getRegister(Reg, VT) };
730 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT,
730 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT,
731 SDValue Glue) {
733 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
738 SDValue getCondCode(ISD::CondCode Cond);
743 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
743 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
743 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
750 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) {
750 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) {
759 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) {
767 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) {
767 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) {
778 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op);
784 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) {
784 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) {
800 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);
804 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT);
804 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT);
808 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
808 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
812 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
812 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
816 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
816 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
820 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
820 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
825 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
825 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
830 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
830 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
835 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);
835 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);
838 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);
838 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);
841 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);
841 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);
846 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset) {
846 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset) {
851 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, SDValue Offset) {
851 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, SDValue Offset) {
851 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, SDValue Offset) {
864 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize,
864 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize,
867 SDValue Ops[] = { Chain,
876 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
876 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
876 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
876 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
877 SDValue InGlue, const SDLoc &DL) {
879 SmallVector<SDValue, 4> Ops;
889 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops);
892 SDValue getUNDEF(EVT VT) {
897 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
903 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
905 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
906 ArrayRef<SDValue> Ops, const SDNodeFlags Flags = SDNodeFlags());
907 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
908 ArrayRef<SDValue> Ops);
909 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
910 ArrayRef<SDValue> Ops);
913 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT);
914 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
914 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
916 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
916 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
917 SDValue N2, const SDNodeFlags Flags = SDNodeFlags());
918 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
918 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
919 SDValue N2, SDValue N3,
919 SDValue N2, SDValue N3,
921 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
921 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
922 SDValue N2, SDValue N3, SDValue N4);
922 SDValue N2, SDValue N3, SDValue N4);
922 SDValue N2, SDValue N3, SDValue N4);
923 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
923 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
924 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
924 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
924 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
924 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
928 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList);
929 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
929 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
930 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
930 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
931 SDValue N2);
932 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
932 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
933 SDValue N2, SDValue N3);
933 SDValue N2, SDValue N3);
934 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
934 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
935 SDValue N2, SDValue N3, SDValue N4);
935 SDValue N2, SDValue N3, SDValue N4);
935 SDValue N2, SDValue N3, SDValue N4);
936 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
936 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
937 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
937 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
937 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
937 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
942 SDValue getStackArgumentTokenFactor(SDValue Chain);
942 SDValue getStackArgumentTokenFactor(SDValue Chain);
944 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
944 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
944 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
944 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
945 SDValue Size, unsigned Align, bool isVol, bool AlwaysInline,
949 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
949 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
949 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
949 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
950 SDValue Size, unsigned Align, bool isVol, bool isTailCall,
954 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
954 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
954 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
954 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
955 SDValue Size, unsigned Align, bool isVol, bool isTailCall,
958 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
958 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
958 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
959 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
960 SDValue Size, Type *SizeTy, unsigned ElemSz,
964 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
964 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
964 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
965 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
966 SDValue Size, Type *SizeTy, unsigned ElemSz,
970 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
970 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
970 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
971 unsigned DstAlign, SDValue Value, SDValue Size,
971 unsigned DstAlign, SDValue Value, SDValue Size,
977 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
977 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
977 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
990 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
990 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
990 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
991 SDValue RHS) {
1002 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
1002 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
1002 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
1002 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
1003 SDValue False, ISD::CondCode Cond) {
1009 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
1009 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
1009 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
1009 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
1012 SDValue simplifyShift(SDValue X, SDValue Y);
1012 SDValue simplifyShift(SDValue X, SDValue Y);
1012 SDValue simplifyShift(SDValue X, SDValue Y);
1016 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y);
1016 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y);
1016 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y);
1020 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1020 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1020 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1021 SDValue SV, unsigned Align);
1027 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1028 SDVTList VTs, SDValue Chain, SDValue Ptr,
1028 SDVTList VTs, SDValue Chain, SDValue Ptr,
1029 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
1029 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
1033 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
1033 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
1034 SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
1034 SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
1038 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
1039 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);
1039 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);
1043 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1044 SDVTList VTList, ArrayRef<SDValue> Ops,
1051 SDValue getMemIntrinsicNode(
1053 ArrayRef<SDValue> Ops, EVT MemVT,
1061 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1062 ArrayRef<SDValue> Ops, EVT MemVT,
1068 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
1068 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
1072 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);
1072 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);
1079 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1079 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1079 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1084 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1084 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1084 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1086 SDValue
1087 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
1088 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
1092 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1093 SDValue Chain, SDValue Ptr, EVT MemVT,
1093 SDValue Chain, SDValue Ptr, EVT MemVT,
1095 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1095 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1095 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1096 SDValue Offset, ISD::MemIndexedMode AM);
1097 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1098 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1098 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1098 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1103 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1104 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1104 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1104 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1111 SDValue
1112 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1112 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1112 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1116 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1116 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1116 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1116 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1118 SDValue
1119 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1119 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1119 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1123 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1123 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1123 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1124 SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
1125 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1125 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1125 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1126 SDValue Offset, ISD::MemIndexedMode AM);
1129 SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, const SDLoc &DL);
1129 SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, const SDLoc &DL);
1131 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1131 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1131 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1132 SDValue Mask, SDValue Src0, EVT MemVT,
1132 SDValue Mask, SDValue Src0, EVT MemVT,
1135 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1135 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1135 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1136 SDValue Ptr, SDValue Mask, EVT MemVT,
1136 SDValue Ptr, SDValue Mask, EVT MemVT,
1139 SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
1140 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1142 SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
1143 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1149 SDValue getTargetMemSDNode(SDVTList VTs, ArrayRef<SDValue> Ops,
1149 SDValue getTargetMemSDNode(SDVTList VTs, ArrayRef<SDValue> Ops,
1154 SDValue getSrcValue(const Value *v);
1157 SDValue getMDNode(const MDNode *MD);
1161 SDValue getBitcast(EVT VT, SDValue V);
1161 SDValue getBitcast(EVT VT, SDValue V);
1164 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
1164 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
1169 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
1169 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
1172 SDValue expandVAArg(SDNode *Node);
1175 SDValue expandVACopy(SDNode *Node);
1181 SDValue getSymbolFunctionGlobalAddress(SDValue Op,
1181 SDValue getSymbolFunctionGlobalAddress(SDValue Op,
1190 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op);
1191 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2);
1191 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2);
1192 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1192 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1193 SDValue Op3);
1194 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1194 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1195 SDValue Op3, SDValue Op4);
1195 SDValue Op3, SDValue Op4);
1196 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1196 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1197 SDValue Op3, SDValue Op4, SDValue Op5);
1197 SDValue Op3, SDValue Op4, SDValue Op5);
1197 SDValue Op3, SDValue Op4, SDValue Op5);
1198 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
1203 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);
1203 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);
1218 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1);
1220 SDValue Op1, SDValue Op2);
1220 SDValue Op1, SDValue Op2);
1222 SDValue Op1, SDValue Op2, SDValue Op3);
1222 SDValue Op1, SDValue Op2, SDValue Op3);
1222 SDValue Op1, SDValue Op2, SDValue Op3);
1224 ArrayRef<SDValue> Ops);
1227 EVT VT2, ArrayRef<SDValue> Ops);
1229 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1231 EVT VT2, SDValue Op1);
1233 EVT VT2, SDValue Op1, SDValue Op2);
1233 EVT VT2, SDValue Op1, SDValue Op2);
1235 ArrayRef<SDValue> Ops);
1240 ArrayRef<SDValue> Ops);
1255 SDValue Op1);
1257 SDValue Op1, SDValue Op2);
1257 SDValue Op1, SDValue Op2);
1259 SDValue Op1, SDValue Op2, SDValue Op3);
1259 SDValue Op1, SDValue Op2, SDValue Op3);
1259 SDValue Op1, SDValue Op2, SDValue Op3);
1261 ArrayRef<SDValue> Ops);
1263 EVT VT2, SDValue Op1, SDValue Op2);
1263 EVT VT2, SDValue Op1, SDValue Op2);
1265 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1265 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1265 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1267 EVT VT2, ArrayRef<SDValue> Ops);
1269 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
1269 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
1271 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
1271 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
1272 SDValue Op3);
1274 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1276 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops);
1278 ArrayRef<SDValue> Ops);
1281 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1282 SDValue Operand);
1285 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1286 SDValue Operand, SDValue Subreg);
1286 SDValue Operand, SDValue Subreg);
1289 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops,
1318 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
1318 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
1345 void ReplaceAllUsesWith(SDValue From, SDValue To);
1345 void ReplaceAllUsesWith(SDValue From, SDValue To);
1347 void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
1351 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
1351 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
1356 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
1356 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
1364 SDValue makeEquivalentMemoryOrdering(LoadSDNode *Old, SDValue New);
1364 SDValue makeEquivalentMemoryOrdering(LoadSDNode *Old, SDValue New);
1434 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);
1438 SDValue CreateStackTemporary(EVT VT1, EVT VT2);
1440 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
1444 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1447 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1451 SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1452 ArrayRef<SDValue> Ops,
1457 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
1458 SDValue N1, SDValue N2);
1458 SDValue N1, SDValue N2);
1461 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
1461 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
1461 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
1470 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits);
1470 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits);
1479 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits,
1479 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits,
1484 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;
1489 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1495 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1500 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
1508 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const;
1515 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
1529 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const;
1529 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const;
1534 bool isKnownToBeAPowerOfTwo(SDValue Val) const;
1543 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;
1553 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
1561 bool isBaseWithConstantOffset(SDValue Op) const;
1566 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
1569 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
1575 bool isKnownNeverZeroFloat(SDValue Op) const;
1578 bool isKnownNeverZero(SDValue Op) const;
1583 bool isEqualTo(SDValue A, SDValue B) const;
1583 bool isEqualTo(SDValue A, SDValue B) const;
1587 bool haveNoCommonBitsSet(SDValue A, SDValue B) const;
1587 bool haveNoCommonBitsSet(SDValue A, SDValue B) const;
1596 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts);
1599 bool isSplatValue(SDValue V, bool AllowUndefs = false);
1602 SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
1602 SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
1606 SDValue getSplatValue(SDValue V);
1606 SDValue getSplatValue(SDValue V);
1616 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
1626 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
1630 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
1630 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
1642 unsigned InferPtrAlignment(SDValue Ptr) const;
1650 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
1650 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
1650 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
1654 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
1654 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
1654 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
1662 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
1662 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
1668 SDValue WidenVector(const SDValue &N, const SDLoc &DL);
1668 SDValue WidenVector(const SDValue &N, const SDLoc &DL);
1672 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
1672 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
1679 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N);
1682 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N);
1686 inline bool isConstantValueOfAnyType(SDValue N) {
1718 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
1719 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1719 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1721 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1769 SDValue SelectionDAG::getTargetMemSDNode(SDVTList VTs,
1770 ArrayRef<SDValue> Ops,
1800 return SDValue(N, 0);
include/llvm/CodeGen/SelectionDAGAddressAnalysis.h 34 SDValue Base;
35 SDValue Index;
41 BaseIndexOffset(SDValue Base, SDValue Index, bool IsIndexSignExt)
41 BaseIndexOffset(SDValue Base, SDValue Index, bool IsIndexSignExt)
43 BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset,
43 BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset,
48 SDValue getBase() { return Base; }
49 SDValue getBase() const { return Base; }
50 SDValue getIndex() { return Index; }
51 SDValue getIndex() const { return Index; }
include/llvm/CodeGen/SelectionDAGISel.h 97 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
99 std::vector<SDValue> &OutOps) {
105 virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const;
111 static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
211 void ReplaceUses(SDValue F, SDValue T) {
211 void ReplaceUses(SDValue F, SDValue T) {
218 void ReplaceUses(const SDValue *F, const SDValue *T, unsigned Num) {
218 void ReplaceUses(const SDValue *F, const SDValue *T, unsigned Num) {
240 void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
254 bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
256 bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
283 const SmallVectorImpl<SDValue> &Operands) const {
287 virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N,
289 SmallVectorImpl<std::pair<SDValue, SDNode*> > &Result) {
293 virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo) {
293 virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo) {
320 ArrayRef<SDValue> Ops, unsigned EmitNodeInfo);
358 void UpdateChains(SDNode *NodeToMatch, SDValue InputChain,
include/llvm/CodeGen/SelectionDAGNodes.h 144 bool operator==(const SDValue &O) const {
147 bool operator!=(const SDValue &O) const {
150 bool operator<(const SDValue &O) const {
157 SDValue getValue(unsigned R) const {
184 inline const SDValue &getOperand(unsigned i) const;
203 bool reachesChainWithoutSideEffects(SDValue Dest,
214 static inline SDValue getEmptyKey() {
215 SDValue V;
220 static inline SDValue getTombstoneKey() {
221 SDValue V;
226 static unsigned getHashValue(const SDValue &Val) {
231 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
231 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
241 static SimpleType getSimplifiedValue(SDValue &Val) {
248 static SimpleType getSimplifiedValue(const SDValue &Val) {
260 SDValue Val;
274 operator const SDValue&() const { return Val; }
278 const SDValue &get() const { return Val; }
294 bool operator==(const SDValue &V) const {
299 bool operator!=(const SDValue &V) const {
304 bool operator<(const SDValue &V) const {
318 inline void set(const SDValue &V);
321 inline void setInitial(const SDValue &V);
913 for (const SDValue &OpV : M->op_values()) {
951 const SDValue &getOperand(unsigned Num) const {
965 std::random_access_iterator_tag, SDValue,
971 const SDValue &operator*() const { return I->get(); }
1135 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1170 inline const SDValue &SDValue::getOperand(unsigned i) const {
1232 inline void SDUse::set(const SDValue &V) {
1238 inline void SDUse::setInitial(const SDValue &V) {
1257 explicit HandleSDNode(SDValue X)
1274 const SDValue &getValue() const { return Op; }
1395 const SDValue &getChain() const { return getOperand(0); }
1396 const SDValue &getBasePtr() const {
1444 const SDValue &getBasePtr() const { return getOperand(1); }
1445 const SDValue &getVal() const { return getOperand(2); }
1662 bool isNullConstant(SDValue V);
1665 bool isNullFPConstant(SDValue V);
1668 bool isAllOnesConstant(SDValue V);
1671 bool isOneConstant(SDValue V);
1675 SDValue peekThroughBitcasts(SDValue V);
1675 SDValue peekThroughBitcasts(SDValue V);
1679 SDValue peekThroughOneUseBitcasts(SDValue V);
1679 SDValue peekThroughOneUseBitcasts(SDValue V);
1683 SDValue peekThroughExtractSubvectors(SDValue V);
1683 SDValue peekThroughExtractSubvectors(SDValue V);
1687 bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1690 ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1695 ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1700 ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1704 ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1710 bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1715 bool isOneOrOneSplat(SDValue V);
1720 bool isAllOnesOrAllOnesSplat(SDValue V);
1950 SDValue getSplatValue(const APInt &DemandedElts,
1957 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
2209 const SDValue &getOffset() const {
2251 const SDValue &getBasePtr() const { return getOperand(1); }
2252 const SDValue &getOffset() const { return getOperand(2); }
2281 const SDValue &getValue() const { return getOperand(1); }
2282 const SDValue &getBasePtr() const { return getOperand(2); }
2283 const SDValue &getOffset() const { return getOperand(3); }
2303 const SDValue &getBasePtr() const {
2306 const SDValue &getMask() const {
2333 const SDValue &getBasePtr() const { return getOperand(1); }
2334 const SDValue &getMask() const { return getOperand(2); }
2335 const SDValue &getPassThru() const { return getOperand(3); }
2368 const SDValue &getValue() const { return getOperand(1); }
2369 const SDValue &getBasePtr() const { return getOperand(2); }
2370 const SDValue &getMask() const { return getOperand(3); }
2409 const SDValue &getBasePtr() const { return getOperand(3); }
2410 const SDValue &getIndex() const { return getOperand(4); }
2411 const SDValue &getMask() const { return getOperand(2); }
2412 const SDValue &getScale() const { return getOperand(5); }
2432 const SDValue &getPassThru() const { return getOperand(1); }
2451 const SDValue &getValue() const { return getOperand(1); }
2654 bool matchUnaryPredicate(SDValue Op,
2663 SDValue LHS, SDValue RHS,
2663 SDValue LHS, SDValue RHS,
include/llvm/CodeGen/SelectionDAGTargetInfo.h 51 virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
52 SDValue Chain, SDValue Op1,
52 SDValue Chain, SDValue Op1,
53 SDValue Op2, SDValue Op3,
53 SDValue Op2, SDValue Op3,
67 virtual SDValue EmitTargetCodeForMemmove(
68 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1,
68 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1,
69 SDValue Op2, SDValue Op3, unsigned Align, bool isVolatile,
69 SDValue Op2, SDValue Op3, unsigned Align, bool isVolatile,
80 virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
81 SDValue Chain, SDValue Op1,
81 SDValue Chain, SDValue Op1,
82 SDValue Op2, SDValue Op3,
82 SDValue Op2, SDValue Op3,
92 virtual std::pair<SDValue, SDValue>
92 virtual std::pair<SDValue, SDValue>
93 EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
94 SDValue Op1, SDValue Op2, SDValue Op3,
94 SDValue Op1, SDValue Op2, SDValue Op3,
94 SDValue Op1, SDValue Op2, SDValue Op3,
104 virtual std::pair<SDValue, SDValue>
104 virtual std::pair<SDValue, SDValue>
105 EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
106 SDValue Src, SDValue Char, SDValue Length,
106 SDValue Src, SDValue Char, SDValue Length,
106 SDValue Src, SDValue Char, SDValue Length,
117 virtual std::pair<SDValue, SDValue>
117 virtual std::pair<SDValue, SDValue>
118 EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
119 SDValue Dest, SDValue Src,
119 SDValue Dest, SDValue Src,
129 virtual std::pair<SDValue, SDValue>
129 virtual std::pair<SDValue, SDValue>
130 EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
131 SDValue Op1, SDValue Op2,
131 SDValue Op1, SDValue Op2,
137 virtual std::pair<SDValue, SDValue>
137 virtual std::pair<SDValue, SDValue>
138 EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
139 SDValue Src, MachinePointerInfo SrcPtrInfo) const {
143 virtual std::pair<SDValue, SDValue>
143 virtual std::pair<SDValue, SDValue>
144 EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
145 SDValue Src, SDValue MaxLength,
145 SDValue Src, SDValue MaxLength,
150 virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl,
151 SDValue Chain, SDValue Addr,
151 SDValue Chain, SDValue Addr,
152 SDValue Size,
include/llvm/CodeGen/TargetLowering.h 179 SDValue Node = SDValue();
341 virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const {
531 virtual bool hasAndNotCompare(SDValue Y) const {
538 virtual bool hasAndNot(SDValue X) const {
548 virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
548 virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
555 virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const {
590 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
590 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
1920 EVT VT, SDValue C) const {
2492 virtual bool isZExtFree(SDValue Val, EVT VT2) const {
2516 virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const { return false; }
2572 virtual bool shouldScalarizeBinop(SDValue VecOp) const {
2969 virtual bool getPreIndexedAddressParts(SDNode * /*N*/, SDValue &/*Base*/,
2970 SDValue &/*Offset*/,
2980 SDValue &/*Base*/,
2981 SDValue &/*Offset*/,
3007 virtual SDValue getPICJumpTableRelocBase(SDValue Table,
3007 virtual SDValue getPICJumpTableRelocBase(SDValue Table,
3021 SDValue &Chain) const;
3023 void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
3024 SDValue &NewRHS, ISD::CondCode &CCCode,
3025 const SDLoc &DL, const SDValue OldLHS,
3026 const SDValue OldRHS) const;
3030 std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
3030 std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
3031 EVT RetVT, ArrayRef<SDValue> Ops,
3041 const SmallVectorImpl<SDValue> &OutVals) const;
3054 SDValue Old;
3055 SDValue New;
3064 bool CombineTo(SDValue O, SDValue N) {
3064 bool CombineTo(SDValue O, SDValue N) {
3089 bool ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
3095 virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
3103 bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &Demanded,
3119 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
3126 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
3133 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask,
3139 SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
3139 SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
3158 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask,
3165 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
3173 virtual void computeKnownBitsForTargetNode(const SDValue Op,
3191 virtual void computeKnownBitsForFrameIndex(const SDValue FIOp,
3201 virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
3212 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef,
3220 virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op,
3230 virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
3231 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3238 SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3238 SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3239 SDValue N1, MutableArrayRef<int> Mask,
3249 virtual bool isKnownNeverNaNForTargetNode(SDValue Op,
3273 SDValue CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo = true);
3273 SDValue CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo = true);
3274 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true);
3274 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true);
3275 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo = true);
3275 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo = true);
3275 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo = true);
3295 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
3295 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
3295 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
3300 virtual SDValue unwrapAddress(SDValue N) const { return N; }
3300 virtual SDValue unwrapAddress(SDValue N) const { return N; }
3320 virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
3369 virtual bool IsDesirableToPromoteOp(SDValue /*Op*/, EVT &/*PVT*/) const {
3405 virtual char isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
3410 virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
3410 virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
3423 virtual SDValue LowerFormalArguments(
3424 SDValue /*Chain*/, CallingConv::ID /*CallConv*/, bool /*isVarArg*/,
3426 SelectionDAG & /*DAG*/, SmallVectorImpl<SDValue> & /*InVals*/) const {
3435 SDValue Chain;
3455 SDValue Callee;
3461 SmallVector<SDValue, 32> OutVals;
3463 SmallVector<SDValue, 4> InVals;
3475 CallLoweringInfo &setChain(SDValue InChain) {
3482 SDValue Target, ArgListTy &&ArgsList) {
3495 SDValue Target, ArgListTy &&ArgsList) {
3505 SDValue Target, ArgListTy &&ArgsList,
3634 std::pair<SDValue, SDValue> LowerCallTo(CallLoweringInfo &CLI) const;
3634 std::pair<SDValue, SDValue> LowerCallTo(CallLoweringInfo &CLI) const;
3641 virtual SDValue
3643 SmallVectorImpl<SDValue> &/*InVals*/) const {
3665 virtual SDValue LowerReturn(SDValue /*Chain*/, CallingConv::ID /*CallConv*/,
3665 virtual SDValue LowerReturn(SDValue /*Chain*/, CallingConv::ID /*CallConv*/,
3668 const SmallVectorImpl<SDValue> & /*OutVals*/,
3679 virtual bool isUsedByReturnOnly(SDNode *, SDValue &/*Chain*/) const {
3747 virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL,
3747 virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL,
3792 SmallVectorImpl<SDValue> &Results,
3800 virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
3800 virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
3812 SmallVectorImpl<SDValue> &/*Results*/,
3827 bool verifyReturnAddressArgumentIsConstant(SDValue Op,
3924 SDValue Op,
3958 virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
3959 std::vector<SDValue> &Ops,
3963 virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
3963 virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
3963 virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
3971 SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization,
3973 SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization,
3980 virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor,
4010 virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
4010 virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
4026 virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
4026 virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
4045 bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS,
4046 SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
4046 SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
4048 SDValue LL = SDValue(), SDValue LH = SDValue(),
4048 SDValue LL = SDValue(), SDValue LH = SDValue(),
4049 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
4049 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
4060 bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
4060 bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
4062 SDValue LL = SDValue(), SDValue LH = SDValue(),
4062 SDValue LL = SDValue(), SDValue LH = SDValue(),
4063 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
4063 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
4069 bool expandFunnelShift(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4075 bool expandROT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4081 bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4087 bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const;
4087 bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const;
4093 bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4096 SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const;
4103 bool expandCTPOP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4110 bool expandCTLZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4117 bool expandCTTZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4125 bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
4130 SDValue scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const;
4135 SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const;
4139 std::pair<SDValue, SDValue> expandUnalignedLoad(LoadSDNode *LD,
4139 std::pair<SDValue, SDValue> expandUnalignedLoad(LoadSDNode *LD,
4144 SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const;
4153 SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL,
4153 SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL,
4153 SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL,
4161 SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
4161 SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
4162 SDValue Index) const;
4166 SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const;
4170 SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
4174 void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
4174 void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
4179 void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
4179 void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
4184 bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
4184 bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
4189 SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const;
4219 virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
4219 virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
4225 virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
4230 virtual SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value, SDValue Addr,
4230 virtual SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value, SDValue Addr,
4230 virtual SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value, SDValue Addr,
4239 SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const;
4239 SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const;
4242 SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4242 SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4242 SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4244 SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4244 SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4244 SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4247 SDValue optimizeSetCCOfSignedTruncationCheck(EVT SCCVT, SDValue N0,
4247 SDValue optimizeSetCCOfSignedTruncationCheck(EVT SCCVT, SDValue N0,
4248 SDValue N1, ISD::CondCode Cond,
4253 SDValue optimizeSetCCByHoistingAndByConstFromLogicalShift(
4254 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4254 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4257 SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
4257 SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
4258 SDValue CompTargetNode, ISD::CondCode Cond,
4261 SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
4261 SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
4261 SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
4265 SDValue prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
4265 SDValue prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
4266 SDValue CompTargetNode, ISD::CondCode Cond,
4269 SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
4269 SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
4269 SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
include/llvm/Support/AlignOf.h 30 T t;
39 template <typename T> union SizerImpl<T> { char arr[sizeof(T)]; };
50 llvm::detail::SizerImpl<T, Ts...>)];
include/llvm/Support/Casting.h 141 template <class X, class Y> LLVM_NODISCARD inline bool isa(const Y &Val) {
142 return isa_impl_wrap<X, const Y,
143 typename simplify_type<const Y>::SimpleType>::doit(Val);
210 To, From, typename simplify_type<From>::SimpleType>::ret_type;
210 To, From, typename simplify_type<From>::SimpleType>::ret_type;
218 static typename cast_retty<To, From>::ret_type doit(From &Val) {
218 static typename cast_retty<To, From>::ret_type doit(From &Val) {
221 simplify_type<From>::getSimplifiedValue(Val));
236 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
236 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
247 inline typename std::enable_if<!is_simple_type<Y>::value,
248 typename cast_retty<X, const Y>::ret_type>::type
249 cast(const Y &Val) {
252 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
252 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
256 inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
256 inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
258 return cast_convert_val<X, Y,
259 typename simplify_type<Y>::SimpleType>::doit(Val);
330 typename std::enable_if<!is_simple_type<Y>::value,
331 typename cast_retty<X, const Y>::ret_type>::type
332 dyn_cast(const Y &Val) {
337 LLVM_NODISCARD inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
337 LLVM_NODISCARD inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
351 typename std::enable_if<!is_simple_type<Y>::value,
352 typename cast_retty<X, const Y>::ret_type>::type
353 dyn_cast_or_null(const Y &Val) {
359 typename std::enable_if<!is_simple_type<Y>::value,
360 typename cast_retty<X, Y>::ret_type>::type
361 dyn_cast_or_null(Y &Val) {
include/llvm/Support/type_traits.h 65 using type = const T &;
91 T t;
122 static auto get(F*) -> decltype(std::declval<F &>() = std::declval<const F &>(), std::true_type{});
122 static auto get(F*) -> decltype(std::declval<F &>() = std::declval<const F &>(), std::true_type{});
122 static auto get(F*) -> decltype(std::declval<F &>() = std::declval<const F &>(), std::true_type{});
130 static auto get(F*) -> decltype(std::declval<F &>() = std::declval<F &&>(), std::true_type{});
130 static auto get(F*) -> decltype(std::declval<F &>() = std::declval<F &&>(), std::true_type{});
130 static auto get(F*) -> decltype(std::declval<F &>() = std::declval<F &&>(), std::true_type{});
145 std::is_copy_constructible<detail::trivial_helper<T>>::value;
147 !std::is_copy_constructible<T>::value;
151 std::is_move_constructible<detail::trivial_helper<T>>::value;
153 !std::is_move_constructible<T>::value;
157 is_copy_assignable<detail::trivial_helper<T>>::value;
159 !is_copy_assignable<T>::value;
163 is_move_assignable<detail::trivial_helper<T>>::value;
165 !is_move_assignable<T>::value;
169 std::is_destructible<detail::trivial_helper<T>>::value;
181 static_assert(value == std::is_trivially_copyable<T>::value,
lib/CodeGen/SelectionDAG/DAGCombiner.cpp 214 SDValue visit(SDNode *N);
271 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
271 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
275 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
275 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
280 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
280 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
280 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
282 SDValue To[] = { Res0, Res1 };
294 bool SimplifyDemandedBits(SDValue Op) {
300 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
310 bool SimplifyDemandedVectorElts(SDValue Op) {
316 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
318 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
323 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
327 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
328 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
329 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
339 SDValue scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
340 SDValue EltNo,
343 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
343 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
344 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
344 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
345 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
345 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
346 SDValue PromoteIntBinOp(SDValue Op);
346 SDValue PromoteIntBinOp(SDValue Op);
347 SDValue PromoteIntShiftOp(SDValue Op);
347 SDValue PromoteIntShiftOp(SDValue Op);
348 SDValue PromoteExtend(SDValue Op);
348 SDValue PromoteExtend(SDValue Op);
349 bool PromoteLoad(SDValue Op);
354 SDValue combine(SDNode *N);
363 SDValue visitTokenFactor(SDNode *N);
364 SDValue visitMERGE_VALUES(SDNode *N);
365 SDValue visitADD(SDNode *N);
366 SDValue visitADDLike(SDNode *N);
367 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
367 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
367 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
368 SDValue visitSUB(SDNode *N);
369 SDValue visitADDSAT(SDNode *N);
370 SDValue visitSUBSAT(SDNode *N);
371 SDValue visitADDC(SDNode *N);
372 SDValue visitADDO(SDNode *N);
373 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
373 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
373 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
374 SDValue visitSUBC(SDNode *N);
375 SDValue visitSUBO(SDNode *N);
376 SDValue visitADDE(SDNode *N);
377 SDValue visitADDCARRY(SDNode *N);
378 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
378 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
378 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
378 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
379 SDValue visitSUBE(SDNode *N);
380 SDValue visitSUBCARRY(SDNode *N);
381 SDValue visitMUL(SDNode *N);
382 SDValue visitMULFIX(SDNode *N);
383 SDValue useDivRem(SDNode *N);
384 SDValue visitSDIV(SDNode *N);
385 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
385 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
385 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
386 SDValue visitUDIV(SDNode *N);
387 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
387 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
387 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
388 SDValue visitREM(SDNode *N);
389 SDValue visitMULHU(SDNode *N);
390 SDValue visitMULHS(SDNode *N);
391 SDValue visitSMUL_LOHI(SDNode *N);
392 SDValue visitUMUL_LOHI(SDNode *N);
393 SDValue visitMULO(SDNode *N);
394 SDValue visitIMINMAX(SDNode *N);
395 SDValue visitAND(SDNode *N);
396 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
396 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
396 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
397 SDValue visitOR(SDNode *N);
398 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
398 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
398 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
399 SDValue visitXOR(SDNode *N);
400 SDValue SimplifyVBinOp(SDNode *N);
401 SDValue visitSHL(SDNode *N);
402 SDValue visitSRA(SDNode *N);
403 SDValue visitSRL(SDNode *N);
404 SDValue visitFunnelShift(SDNode *N);
405 SDValue visitRotate(SDNode *N);
406 SDValue visitABS(SDNode *N);
407 SDValue visitBSWAP(SDNode *N);
408 SDValue visitBITREVERSE(SDNode *N);
409 SDValue visitCTLZ(SDNode *N);
410 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
411 SDValue visitCTTZ(SDNode *N);
412 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
413 SDValue visitCTPOP(SDNode *N);
414 SDValue visitSELECT(SDNode *N);
415 SDValue visitVSELECT(SDNode *N);
416 SDValue visitSELECT_CC(SDNode *N);
417 SDValue visitSETCC(SDNode *N);
418 SDValue visitSETCCCARRY(SDNode *N);
419 SDValue visitSIGN_EXTEND(SDNode *N);
420 SDValue visitZERO_EXTEND(SDNode *N);
421 SDValue visitANY_EXTEND(SDNode *N);
422 SDValue visitAssertExt(SDNode *N);
423 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
424 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
425 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
426 SDValue visitTRUNCATE(SDNode *N);
427 SDValue visitBITCAST(SDNode *N);
428 SDValue visitBUILD_PAIR(SDNode *N);
429 SDValue visitFADD(SDNode *N);
430 SDValue visitFSUB(SDNode *N);
431 SDValue visitFMUL(SDNode *N);
432 SDValue visitFMA(SDNode *N);
433 SDValue visitFDIV(SDNode *N);
434 SDValue visitFREM(SDNode *N);
435 SDValue visitFSQRT(SDNode *N);
436 SDValue visitFCOPYSIGN(SDNode *N);
437 SDValue visitFPOW(SDNode *N);
438 SDValue visitSINT_TO_FP(SDNode *N);
439 SDValue visitUINT_TO_FP(SDNode *N);
440 SDValue visitFP_TO_SINT(SDNode *N);
441 SDValue visitFP_TO_UINT(SDNode *N);
442 SDValue visitFP_ROUND(SDNode *N);
443 SDValue visitFP_EXTEND(SDNode *N);
444 SDValue visitFNEG(SDNode *N);
445 SDValue visitFABS(SDNode *N);
446 SDValue visitFCEIL(SDNode *N);
447 SDValue visitFTRUNC(SDNode *N);
448 SDValue visitFFLOOR(SDNode *N);
449 SDValue visitFMINNUM(SDNode *N);
450 SDValue visitFMAXNUM(SDNode *N);
451 SDValue visitFMINIMUM(SDNode *N);
452 SDValue visitFMAXIMUM(SDNode *N);
453 SDValue visitBRCOND(SDNode *N);
454 SDValue visitBR_CC(SDNode *N);
455 SDValue visitLOAD(SDNode *N);
457 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
457 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
458 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
460 SDValue visitSTORE(SDNode *N);
461 SDValue visitLIFETIME_END(SDNode *N);
462 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
463 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
464 SDValue visitBUILD_VECTOR(SDNode *N);
465 SDValue visitCONCAT_VECTORS(SDNode *N);
466 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
467 SDValue visitVECTOR_SHUFFLE(SDNode *N);
468 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
469 SDValue visitINSERT_SUBVECTOR(SDNode *N);
470 SDValue visitMLOAD(SDNode *N);
471 SDValue visitMSTORE(SDNode *N);
472 SDValue visitMGATHER(SDNode *N);
473 SDValue visitMSCATTER(SDNode *N);
474 SDValue visitFP_TO_FP16(SDNode *N);
475 SDValue visitFP16_TO_FP(SDNode *N);
476 SDValue visitVECREDUCE(SDNode *N);
478 SDValue visitFADDForFMACombine(SDNode *N);
479 SDValue visitFSUBForFMACombine(SDNode *N);
480 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
482 SDValue XformToShuffleWithZero(SDNode *N);
484 const SDLoc &DL, SDValue N0,
485 SDValue N1);
486 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
486 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
487 SDValue N1);
488 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
488 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
489 SDValue N1, SDNodeFlags Flags);
491 SDValue visitShiftByConstant(SDNode *N);
493 SDValue foldSelectOfConstants(SDNode *N);
494 SDValue foldVSelectOfConstants(SDNode *N);
495 SDValue foldBinOpIntoSelect(SDNode *BO);
496 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
496 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
497 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
498 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
498 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
498 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
498 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
499 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
499 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
499 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
500 SDValue N2, SDValue N3, ISD::CondCode CC,
500 SDValue N2, SDValue N3, ISD::CondCode CC,
502 SDValue convertSelectOfFPConstantsToLoadOffset(
503 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
503 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
503 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
503 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
505 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
505 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
505 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
506 SDValue N2, SDValue N3, ISD::CondCode CC);
506 SDValue N2, SDValue N3, ISD::CondCode CC);
507 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
507 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
507 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
509 SDValue unfoldMaskedMerge(SDNode *N);
510 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
511 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
511 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
511 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
513 SDValue rebuildSetCC(SDValue N);
513 SDValue rebuildSetCC(SDValue N);
515 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
515 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
515 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
516 SDValue &CC) const;
517 bool isOneUseSetCC(SDValue N) const;
518 bool isCheaperToUseNegatedFPOps(SDValue X, SDValue Y);
518 bool isCheaperToUseNegatedFPOps(SDValue X, SDValue Y);
520 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
522 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
523 SDValue CombineExtLoad(SDNode *N);
524 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
525 SDValue combineRepeatedFPDivisors(SDNode *N);
526 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
527 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
528 SDValue BuildSDIV(SDNode *N);
529 SDValue BuildSDIVPow2(SDNode *N);
530 SDValue BuildUDIV(SDNode *N);
531 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
531 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
532 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
532 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
532 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
533 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
533 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
534 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
534 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
535 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
535 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
536 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
536 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
536 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
538 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
538 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
538 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
540 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
540 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
540 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
542 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
542 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
542 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
543 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
543 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
543 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
543 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
544 SDValue InnerPos, SDValue InnerNeg,
544 SDValue InnerPos, SDValue InnerNeg,
547 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
547 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
547 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
548 SDValue MatchLoadCombine(SDNode *N);
549 SDValue MatchStoreCombine(StoreSDNode *N);
550 SDValue ReduceLoadWidth(SDNode *N);
551 SDValue ReduceLoadOpStoreWidth(SDNode *N);
552 SDValue splitMergedValStore(StoreSDNode *ST);
553 SDValue TransformFPLoadStorePair(SDNode *N);
554 SDValue convertBuildVecZextToZext(SDNode *N);
555 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
556 SDValue reduceBuildVecToShuffle(SDNode *N);
557 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
558 ArrayRef<int> VectorMask, SDValue VecIn1,
559 SDValue VecIn2, unsigned LeftIdx,
561 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
565 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
566 SmallVectorImpl<SDValue> &Aliases);
573 SDValue FindBetterChain(SDNode *N, SDValue Chain);
573 SDValue FindBetterChain(SDNode *N, SDValue Chain);
605 SDValue &AddNode,
606 SDValue &ConstNode);
629 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
671 SDValue distributeTruncateThroughAnd(SDNode *N);
709 SDValue OrigLoad, SDValue ExtLoad,
709 SDValue OrigLoad, SDValue ExtLoad,
749 SDValue TargetLowering::DAGCombinerInfo::
750 CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
754 SDValue TargetLowering::DAGCombinerInfo::
755 CombineTo(SDNode *N, SDValue Res, bool AddTo) {
759 SDValue TargetLowering::DAGCombinerInfo::
760 CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
760 CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
783 for (const SDValue &Op : N->ops())
807 bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
807 bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
807 bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
808 SDValue &CC) const {
834 bool DAGCombiner::isOneUseSetCC(SDValue N) const {
835 SDValue N0, N1, N2;
843 static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
854 static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
860 for (const SDValue &Op : N->op_values()) {
873 static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
882 SDValue N0,
883 SDValue N1) {
938 SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
939 SDValue N0, SDValue N1) {
939 SDValue N0, SDValue N1) {
952 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, C1, C2))
959 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
969 SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
969 SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
970 SDValue N1, SDNodeFlags Flags) {
982 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1))
984 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0))
989 SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
989 SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1041 bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1064 bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1089 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1100 SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1100 SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1118 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1122 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1137 SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1137 SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1143 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1154 SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1154 SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1158 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1171 SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1171 SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1194 SDValue N0 = Op.getOperand(0);
1195 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1198 SDValue N1 = Op.getOperand(1);
1199 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1202 SDValue RV =
1236 SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1236 SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1259 SDValue N0 = Op.getOperand(0);
1260 SDValue N1 = Op.getOperand(1);
1272 SDValue RV =
1285 SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1285 SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1313 bool DAGCombiner::PromoteLoad(SDValue Op) {
1342 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1345 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1377 for (const SDValue &ChildN : N->op_values())
1440 for (const SDValue &ChildN : N->op_values())
1444 SDValue RV = combine(N);
1488 SDValue DAGCombiner::visit(SDNode *N) {
1622 SDValue DAGCombiner::combine(SDNode *N) {
1623 SDValue RV = visit(N);
1674 SDValue N0 = N->getOperand(0);
1675 SDValue N1 = N->getOperand(1);
1679 SDValue Ops[] = {N1, N0};
1692 static SDValue getInputChainForNode(SDNode *N) {
1705 SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1726 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1750 for (const SDValue &Op : TF->op_values()) {
1796 for (const SDValue &Op : Ops) {
1846 for (const SDValue &Op : CurNode->op_values())
1867 SDValue Result;
1873 SmallVector<SDValue, 8> PrunedOps;
1875 for (const SDValue &Op : Ops) {
1890 SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1900 SmallVector<SDValue, 8> Ops;
1911 static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1916 SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1924 SDValue Sel = BO->getOperand(0);
1933 SDValue CT = Sel.getOperand(1);
1938 SDValue CF = Sel.getOperand(2);
1954 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
1975 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
1982 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
1989 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1994 static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
2002 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
2003 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
2014 SDValue SetCC = Z.getOperand(0);
2027 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
2028 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
2035 static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2043 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2044 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2050 SDValue Not = ShiftOp.getOperand(0);
2056 SDValue ShAmt = ShiftOp.getOperand(1);
2066 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
2074 SDValue DAGCombiner::visitADDLike(SDNode *N) {
2075 SDValue N0 = N->getOperand(0);
2076 SDValue N1 = N->getOperand(1);
2082 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2116 SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N1.getNode(),
2125 SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N1.getNode(),
2137 SDValue X = N0.getOperand(0);
2142 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2152 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2157 if (SDValue NewSel = foldBinOpIntoSelect(N))
2162 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
2214 SDValue N00 = N0.getOperand(0);
2215 SDValue N01 = N0.getOperand(1);
2216 SDValue N10 = N1.getOperand(0);
2217 SDValue N11 = N1.getOperand(1);
2250 SDValue A, Xor;
2270 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2279 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2283 if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
2286 if (SDValue Combined = visitADDLikeCommutative(N1, N0, N))
2292 SDValue DAGCombiner::visitADD(SDNode *N) {
2293 SDValue N0 = N->getOperand(0);
2294 SDValue N1 = N->getOperand(1);
2298 if (SDValue Combined = visitADDLike(N))
2301 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2304 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2315 SDValue DAGCombiner::visitADDSAT(SDNode *N) {
2317 SDValue N0 = N->getOperand(0);
2318 SDValue N1 = N->getOperand(1);
2358 static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2358 static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2403 static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2403 static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2403 static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2418 SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2418 SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2418 SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2431 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2440 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2450 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2457 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2467 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2475 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2489 if (SDValue Carry = getAsCarry(TLI, N1))
2497 SDValue DAGCombiner::visitADDC(SDNode *N) {
2498 SDValue N0 = N->getOperand(0);
2499 SDValue N1 = N->getOperand(1);
2527 static SDValue flipBoolean(SDValue V, const SDLoc &DL,
2527 static SDValue flipBoolean(SDValue V, const SDLoc &DL,
2531 SDValue Cst;
2552 static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2552 static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2587 SDValue DAGCombiner::visitADDO(SDNode *N) {
2588 SDValue N0 = N->getOperand(0);
2589 SDValue N1 = N->getOperand(1);
2618 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2624 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2627 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2634 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2634 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2634 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2642 SDValue Y = N1.getOperand(0);
2643 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2651 if (SDValue Carry = getAsCarry(TLI, N1))
2658 SDValue DAGCombiner::visitADDE(SDNode *N) {
2659 SDValue N0 = N->getOperand(0);
2660 SDValue N1 = N->getOperand(1);
2661 SDValue CarryIn = N->getOperand(2);
2677 SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2678 SDValue N0 = N->getOperand(0);
2679 SDValue N1 = N->getOperand(1);
2680 SDValue CarryIn = N->getOperand(2);
2700 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2707 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2710 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2738 static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2739 SDValue X, SDValue Carry0, SDValue Carry1,
2739 SDValue X, SDValue Carry0, SDValue Carry1,
2739 SDValue X, SDValue Carry0, SDValue Carry1,
2746 SDValue Z;
2767 SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
2803 SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2803 SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2803 SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2803 SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2807 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
2809 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
2831 if (auto Y = getAsCarry(TLI, N1)) {
2833 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
2835 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
2844 static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2853 SDValue DAGCombiner::visitSUB(SDNode *N) {
2854 SDValue N0 = N->getOperand(0);
2855 SDValue N1 = N->getOperand(1);
2861 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2880 if (SDValue NewSel = foldBinOpIntoSelect(N))
2945 SDValue NewC = DAG.FoldConstantArithmetic(
2953 SDValue N11 = N1.getOperand(1);
2956 SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2967 SDValue NewC = DAG.FoldConstantArithmetic(
2977 SDValue NewC = DAG.FoldConstantArithmetic(
3013 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3020 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3033 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3036 if (SDValue V = foldAddSubOfSignBit(N, DAG))
3039 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3044 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3054 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3062 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3068 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3075 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3081 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3092 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3099 SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
3100 SDValue S0 = N1.getOperand(0);
3129 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3138 SDValue ShAmt = N1.getOperand(1);
3142 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3149 if (SDValue Carry = getAsCarry(TLI, N0)) {
3150 SDValue X = N1;
3151 SDValue Zero = DAG.getConstant(0, DL, VT);
3152 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3162 SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
3163 SDValue N0 = N->getOperand(0);
3164 SDValue N1 = N->getOperand(1);
3199 SDValue DAGCombiner::visitSUBC(SDNode *N) {
3200 SDValue N0 = N->getOperand(0);
3201 SDValue N1 = N->getOperand(1);
3227 SDValue DAGCombiner::visitSUBO(SDNode *N) {
3228 SDValue N0 = N->getOperand(0);
3229 SDValue N1 = N->getOperand(1);
3266 SDValue DAGCombiner::visitSUBE(SDNode *N) {
3267 SDValue N0 = N->getOperand(0);
3268 SDValue N1 = N->getOperand(1);
3269 SDValue CarryIn = N->getOperand(2);
3278 SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
3279 SDValue N0 = N->getOperand(0);
3280 SDValue N1 = N->getOperand(1);
3281 SDValue CarryIn = N->getOperand(2);
3295 SDValue DAGCombiner::visitMULFIX(SDNode *N) {
3296 SDValue N0 = N->getOperand(0);
3297 SDValue N1 = N->getOperand(1);
3298 SDValue Scale = N->getOperand(2);
3317 SDValue DAGCombiner::visitMUL(SDNode *N) {
3318 SDValue N0 = N->getOperand(0);
3319 SDValue N1 = N->getOperand(1);
3333 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3373 if (SDValue NewSel = foldBinOpIntoSelect(N))
3387 SDValue LogBase2 = BuildLogBase2(N1, DL);
3389 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3429 SDValue Shl =
3431 SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0);
3442 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
3450 SDValue Sh(nullptr, 0), Y(nullptr, 0);
3464 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
3481 if (SDValue RMUL = reassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags()))
3507 SDValue DAGCombiner::useDivRem(SDNode *Node) {
3541 SDValue Op0 = Node->getOperand(0);
3542 SDValue Op1 = Node->getOperand(1);
3543 SDValue combined;
3577 static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
3578 SDValue N0 = N->getOperand(0);
3579 SDValue N1 = N->getOperand(1);
3623 SDValue DAGCombiner::visitSDIV(SDNode *N) {
3624 SDValue N0 = N->getOperand(0);
3625 SDValue N1 = N->getOperand(1);
3631 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3650 if (SDValue V = simplifyDivRem(N, DAG))
3653 if (SDValue NewSel = foldBinOpIntoSelect(N))
3661 if (SDValue V = visitSDIVLike(N0, N1, N)) {
3666 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3667 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3680 if (SDValue DivRem = useDivRem(N))
3686 SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3686 SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3686 SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3710 if (SDValue Res = BuildSDIVPow2(N))
3715 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
3716 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
3718 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
3723 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
3728 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
3730 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
3732 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
3737 SDValue One = DAG.getConstant(1, DL, VT);
3738 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
3739 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
3740 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
3741 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
3746 SDValue Zero = DAG.getConstant(0, DL, VT);
3747 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
3750 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
3751 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
3761 if (SDValue Op = BuildSDIV(N))
3767 SDValue DAGCombiner::visitUDIV(SDNode *N) {
3768 SDValue N0 = N->getOperand(0);
3769 SDValue N1 = N->getOperand(1);
3775 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3784 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
3793 if (SDValue V = simplifyDivRem(N, DAG))
3796 if (SDValue NewSel = foldBinOpIntoSelect(N))
3799 if (SDValue V = visitUDIVLike(N0, N1, N)) {
3804 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3805 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3818 if (SDValue DivRem = useDivRem(N))
3824 SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3824 SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3824 SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3831 SDValue LogBase2 = BuildLogBase2(N1, DL);
3835 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3842 SDValue N10 = N1.getOperand(0);
3845 SDValue LogBase2 = BuildLogBase2(N10, DL);
3849 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3851 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3861 if (SDValue Op = BuildUDIV(N))
3868 SDValue DAGCombiner::visitREM(SDNode *N) {
3870 SDValue N0 = N->getOperand(0);
3871 SDValue N1 = N->getOperand(1);
3882 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3889 if (SDValue V = simplifyDivRem(N, DAG))
3892 if (SDValue NewSel = foldBinOpIntoSelect(N))
3901 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3904 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3911 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3927 SDValue OptimizedDiv =
3935 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3936 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3944 if (SDValue DivRem = useDivRem(N))
3950 SDValue DAGCombiner::visitMULHS(SDNode *N) {
3951 SDValue N0 = N->getOperand(0);
3952 SDValue N1 = N->getOperand(1);
3997 SDValue DAGCombiner::visitMULHU(SDNode *N) {
3998 SDValue N0 = N->getOperand(0);
3999 SDValue N1 = N->getOperand(1);
4025 SDValue LogBase2 = BuildLogBase2(N1, DL);
4026 SDValue SRLAmt = DAG.getNode(
4029 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4056 SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
4062 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4070 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4080 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4082 SDValue LoOpt = combine(Lo.getNode());
4090 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4092 SDValue HiOpt = combine(Hi.getNode());
4102 SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
4103 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
4116 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
4117 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
4133 SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
4134 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
4142 SDValue Zero = DAG.getConstant(0, DL, VT);
4148 SDValue Zero = DAG.getConstant(0, DL, VT);
4159 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
4160 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
4176 SDValue DAGCombiner::visitMULO(SDNode *N) {
4177 SDValue N0 = N->getOperand(0);
4178 SDValue N1 = N->getOperand(1);
4204 SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
4205 SDValue N0 = N->getOperand(0);
4206 SDValue N1 = N->getOperand(1);
4211 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4249 SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
4250 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
4266 SDValue X = N0.getOperand(0);
4267 SDValue Y = N1.getOperand(0);
4290 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4312 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4324 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4333 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4350 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4383 SDValue ShOp = N0.getOperand(1);
4389 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
4402 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
4412 SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4412 SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4412 SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4414 SDValue LL, LR, RL, RR, N0CC, N1CC;
4457 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
4476 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
4488 SDValue One = DAG.getConstant(1, DL, OpVT);
4489 SDValue Two = DAG.getConstant(2, DL, OpVT);
4490 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
4502 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
4503 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
4504 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
4505 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4526 SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
4527 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
4528 SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
4529 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
4530 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4562 SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4562 SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4562 SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4570 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
4592 SDValue NewAdd =
4636 SDValue SL(N0);
4641 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
4644 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
4645 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
4646 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
4647 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
4777 for (SDValue Op : N->op_values()) {
4879 SDValue MaskOp = N->getOperand(1);
4885 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
4895 SDValue Op0 = LogicN->getOperand(0);
4896 SDValue Op1 = LogicN->getOperand(1);
4901 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
4910 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
4916 SDValue NewLoad = ReduceLoadWidth(And.getNode());
4932 SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
4935 SDValue N0 = N->getOperand(0);
4936 SDValue N1 = N->getOperand(1);
4945 SDValue Y; // Shift amount.
4962 SDValue X;
4974 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
4976 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
4984 static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
4995 SDValue Not = And->getOperand(0), And1 = And->getOperand(1);
5004 SDValue Srl = Not.getOperand(0);
5024 SDValue X = DAG.getZExtOrTrunc(Srl.getOperand(0), DL, VT);
5026 SDValue Mask = DAG.getConstant(
5028 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask);
5029 SDValue Zero = DAG.getConstant(0, DL, VT);
5030 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
5034 SDValue DAGCombiner::visitAND(SDNode *N) {
5035 SDValue N0 = N->getOperand(0);
5036 SDValue N1 = N->getOperand(1);
5045 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5083 if (SDValue NewSel = foldBinOpIntoSelect(N))
5087 if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
5092 if (SDValue Shuffle = XformToShuffleWithZero(N))
5104 SDValue N0Op0 = N0.getOperand(0);
5108 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5200 SDValue NewLoad(Load, 0);
5214 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
5232 if (SDValue Res = ReduceLoadWidth(N)) {
5251 if (SDValue Combined = visitANDLike(N0, N1, N))
5256 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
5268 SDValue SubRHS = N0.getOperand(1);
5298 SDValue ExtLoad =
5309 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
5314 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
5318 if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
5325 SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5325 SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5325 SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5381 SDValue N00 = N0->getOperand(0);
5392 SDValue N10 = N1->getOperand(0);
5427 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
5443 static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
5451 SDValue N0 = N.getOperand(0);
5528 static bool isBSwapHWordPair(SDValue N, MutableArrayRef<SDNode *> Parts) {
5550 SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
5550 SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
5550 SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
5574 SDValue N00 = N0.getOperand(0);
5575 SDValue N01 = N0.getOperand(1);
5587 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
5592 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
5604 SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
5604 SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
5604 SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
5612 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
5632 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5647 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5656 static SDValue visitORCommutative(
5657 SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N) {
5657 SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N) {
5672 SDValue DAGCombiner::visitOR(SDNode *N) {
5673 SDValue N0 = N->getOperand(0);
5674 SDValue N1 = N->getOperand(1);
5683 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5751 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
5752 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
5754 SDValue LegalShuffle =
5780 if (SDValue NewSel = foldBinOpIntoSelect(N))
5787 if (SDValue Combined = visitORLike(N0, N1, N))
5791 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
5793 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
5797 if (SDValue ROR = reassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags()))
5807 if (SDValue COR = DAG.FoldConstantArithmetic(
5809 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
5815 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
5817 if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
5822 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
5826 if (SDValue Rot = MatchRotate(N0, N1, SDLoc(N)))
5829 if (SDValue Load = MatchLoadCombine(N))
5839 if (SDValue Combined = visitADDLike(N))
5845 static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
5845 static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
5845 static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
5855 static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
5855 static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
5856 SDValue &Mask) {
5889 static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
5889 static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
5890 SDValue ExtractFrom, SDValue &Mask,
5890 SDValue ExtractFrom, SDValue &Mask,
5900 SDValue OppShiftLHS = OppShift.getOperand(0);
5991 SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
6004 static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
6004 static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
6056 SDValue NegOp1 = Neg.getOperand(1);
6112 SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
6112 SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
6112 SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
6113 SDValue Neg, SDValue InnerPos,
6113 SDValue Neg, SDValue InnerPos,
6114 SDValue InnerNeg, unsigned PosOpcode,
6136 SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
6136 SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
6136 SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
6152 if (SDValue Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
6158 SDValue LHSShift; // The shift.
6159 SDValue LHSMask; // AND value if any.
6162 SDValue RHSShift; // The shift.
6163 SDValue RHSMask; // AND value if any.
6180 if (SDValue NewRHSShift =
6185 if (SDValue NewLHSShift =
6209 SDValue LHSShiftArg = LHSShift.getOperand(0);
6210 SDValue LHSShiftAmt = LHSShift.getOperand(1);
6211 SDValue RHSShiftArg = RHSShift.getOperand(0);
6212 SDValue RHSShiftAmt = RHSShift.getOperand(1);
6221 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
6226 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
6227 SDValue Mask = AllOnes;
6230 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
6235 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
6252 SDValue LExtOp0 = LHSShiftAmt;
6253 SDValue RExtOp0 = RHSShiftAmt;
6266 SDValue TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
6271 SDValue TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
6324 calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
6374 SDValue NarrowOp = Op->getOperand(0);
6442 static SDValue stripTruncAndExt(SDValue Value) {
6442 static SDValue stripTruncAndExt(SDValue Value) {
6475 SDValue DAGCombiner::MatchStoreCombine(StoreSDNode *N) {
6477 SDValue Chain;
6501 SDValue CombinedValue;
6509 SDValue Trunc = Store->getValue();
6515 SDValue Value = Trunc.getOperand(0);
6609 SDValue NewStore =
6648 SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
6678 SDValue Chain;
6699 SDValue LChain = L->getChain();
6763 SDValue NewLoad =
6787 SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
6798 SDValue X, Y, M;
6802 SDValue Xor = And.getOperand(XorIdx);
6805 SDValue Xor0 = Xor.getOperand(0);
6806 SDValue Xor1 = Xor.getOperand(1);
6820 SDValue N0 = N->getOperand(0);
6821 SDValue N1 = N->getOperand(1);
6842 SDValue NotX = DAG.getNOT(DL, X, VT);
6843 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
6844 SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
6845 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
6849 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
6850 SDValue NotM = DAG.getNOT(DL, M, VT);
6851 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
6856 SDValue DAGCombiner::visitXOR(SDNode *N) {
6857 SDValue N0 = N->getOperand(0);
6858 SDValue N1 = N->getOperand(1);
6863 if (SDValue FoldedVOp = SimplifyVBinOp(N))
6895 if (SDValue NewSel = foldBinOpIntoSelect(N))
6899 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags()))
6904 SDValue LHS, RHS, CC;
6925 SDValue V = N0.getOperand(0);
6936 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
6948 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
6969 SDValue X = N0.getOperand(0);
6970 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
6990 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
6999 SDValue A = N0Opcode == ISD::ADD ? N0 : N1;
7000 SDValue S = N0Opcode == ISD::SRA ? N0 : N1;
7002 SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
7003 SDValue S0 = S.getOperand(0);
7043 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
7047 if (SDValue MM = unfoldMaskedMerge(N))
7061 static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG) {
7063 SDValue LogicOp = Shift->getOperand(0);
7074 SDValue C1 = Shift->getOperand(1);
7104 SDValue X, Y;
7117 SDValue ShiftSumC = DAG.getConstant(*C0Val + C1Val, DL, ShiftAmtVT);
7118 SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC);
7119 SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1);
7129 SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
7137 SDValue LHS = N->getOperand(0);
7146 if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
7174 SDValue BinOpLHSVal = LHS.getOperand(0);
7191 SDValue NewRHS = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(1),
7195 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
7200 SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
7208 SDValue N01 = N->getOperand(0).getOperand(1);
7211 SDValue N00 = N->getOperand(0).getOperand(0);
7212 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
7213 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
7223 SDValue DAGCombiner::visitRotate(SDNode *N) {
7225 SDValue N0 = N->getOperand(0);
7226 SDValue N1 = N->getOperand(1);
7254 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
7267 if (SDValue CombinedShift =
7269 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
7270 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
7281 SDValue DAGCombiner::visitSHL(SDNode *N) {
7282 SDValue N0 = N->getOperand(0);
7283 SDValue N1 = N->getOperand(1);
7284 if (SDValue V = DAG.simplifyShift(N0, N1))
7293 if (SDValue FoldedVOp = SimplifyVBinOp(N))
7301 SDValue N00 = N0->getOperand(0);
7302 SDValue N01 = N0->getOperand(1);
7308 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
7324 if (SDValue NewSel = foldBinOpIntoSelect(N))
7335 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
7364 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7378 SDValue N0Op0 = N0.getOperand(0);
7379 SDValue InnerShiftAmt = N0Op0.getOperand(1);
7408 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0));
7409 SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT);
7420 SDValue N0Op0 = N0.getOperand(0);
7421 SDValue InnerShiftAmt = N0Op0.getOperand(1);
7434 SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT);
7471 SDValue Shift;
7494 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
7495 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
7508 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
7509 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7519 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7525 if (SDValue NewSHL = visitShiftByConstant(N))
7531 SDValue DAGCombiner::visitSRA(SDNode *N) {
7532 SDValue N0 = N->getOperand(0);
7533 SDValue N1 = N->getOperand(1);
7534 if (SDValue V = DAG.simplifyShift(N0, N1))
7548 if (SDValue FoldedVOp = SimplifyVBinOp(N))
7559 if (SDValue NewSel = foldBinOpIntoSelect(N))
7582 SmallVector<SDValue, 16> ShiftValues;
7595 SDValue ShiftValue;
7632 SDValue Amt = DAG.getConstant(ShiftAmt, DL,
7634 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
7636 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
7651 SDValue Shl = N0.getOperand(0);
7666 SDValue Trunc = DAG.getZExtOrTrunc(Shl.getOperand(0), DL, TruncVT);
7667 SDValue ShiftC = DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).
7669 SDValue Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC);
7678 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
7691 SDValue N0Op0 = N0.getOperand(0);
7697 SDValue Amt = DAG.getConstant(N1C->getZExtValue() + TruncBits, DL,
7699 SDValue SRA =
7716 if (SDValue NewSRA = visitShiftByConstant(N))
7722 SDValue DAGCombiner::visitSRL(SDNode *N) {
7723 SDValue N0 = N->getOperand(0);
7724 SDValue N1 = N->getOperand(1);
7725 if (SDValue V = DAG.simplifyShift(N0, N1))
7733 if (SDValue FoldedVOp = SimplifyVBinOp(N))
7744 if (SDValue NewSel = foldBinOpIntoSelect(N))
7774 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7808 SDValue Mask =
7826 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
7867 SDValue Op = N0.getOperand(0);
7886 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
7897 if (SDValue NewSRL = visitShiftByConstant(N))
7901 if (SDValue NarrowLoad = ReduceLoadWidth(N))
7936 SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
7938 SDValue N0 = N->getOperand(0);
7939 SDValue N1 = N->getOperand(1);
7940 SDValue N2 = N->getOperand(2);
8011 SDValue DAGCombiner::visitABS(SDNode *N) {
8012 SDValue N0 = N->getOperand(0);
8027 SDValue DAGCombiner::visitBSWAP(SDNode *N) {
8028 SDValue N0 = N->getOperand(0);
8040 SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
8041 SDValue N0 = N->getOperand(0);
8053 SDValue DAGCombiner::visitCTLZ(SDNode *N) {
8054 SDValue N0 = N->getOperand(0);
8070 SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
8071 SDValue N0 = N->getOperand(0);
8080 SDValue DAGCombiner::visitCTTZ(SDNode *N) {
8081 SDValue N0 = N->getOperand(0);
8097 SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
8098 SDValue N0 = N->getOperand(0);
8107 SDValue DAGCombiner::visitCTPOP(SDNode *N) {
8108 SDValue N0 = N->getOperand(0);
8119 static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS,
8120 SDValue RHS,
8131 static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
8131 static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
8132 SDValue RHS, SDValue True, SDValue False,
8132 SDValue RHS, SDValue True, SDValue False,
8132 SDValue RHS, SDValue True, SDValue False,
8180 static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG) {
8181 SDValue Cond = N->getOperand(0);
8182 SDValue C1 = N->getOperand(1);
8183 SDValue C2 = N->getOperand(2);
8194 SDValue X = Cond.getOperand(0);
8195 SDValue CondC = Cond.getOperand(1);
8201 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
8202 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
8208 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
8209 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
8215 SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
8216 SDValue Cond = N->getOperand(0);
8217 SDValue N1 = N->getOperand(1);
8218 SDValue N2 = N->getOperand(2);
8238 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8245 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8287 SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT);
8291 if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8313 SDValue NotCond =
8323 SDValue DAGCombiner::visitSELECT(SDNode *N) {
8324 SDValue N0 = N->getOperand(0);
8325 SDValue N1 = N->getOperand(1);
8326 SDValue N2 = N->getOperand(2);
8332 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8340 if (SDValue V = foldSelectOfConstants(N))
8345 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8351 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8378 SDValue Cond0 = N0->getOperand(0);
8379 SDValue Cond1 = N0->getOperand(1);
8380 SDValue InnerSelect =
8391 SDValue Cond0 = N0->getOperand(0);
8392 SDValue Cond1 = N0->getOperand(1);
8393 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(),
8405 SDValue N1_0 = N1->getOperand(0);
8406 SDValue N1_1 = N1->getOperand(1);
8407 SDValue N1_2 = N1->getOperand(2);
8411 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
8416 if (SDValue Combined = visitANDLike(N0, N1_0, N)) {
8424 SDValue N2_0 = N2->getOperand(0);
8425 SDValue N2_1 = N2->getOperand(1);
8426 SDValue N2_2 = N2->getOperand(2);
8430 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
8435 if (SDValue Combined = visitORLike(N0, N2_0, N))
8443 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
8444 SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1);
8451 SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1);
8459 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2,
8487 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
8498 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1,
8512 static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
8514 SDValue Cond = N->getOperand(0);
8515 SDValue LHS = N->getOperand(1);
8516 SDValue RHS = N->getOperand(2);
8565 SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
8567 SDValue Mask = MSC->getMask();
8568 SDValue Chain = MSC->getChain();
8578 SDValue DAGCombiner::visitMSTORE(SDNode *N) {
8580 SDValue Mask = MST->getMask();
8581 SDValue Chain = MST->getChain();
8591 SDValue DAGCombiner::visitMGATHER(SDNode *N) {
8593 SDValue Mask = MGT->getMask();
8603 SDValue DAGCombiner::visitMLOAD(SDNode *N) {
8605 SDValue Mask = MLD->getMask();
8617 SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
8618 SDValue Cond = N->getOperand(0);
8619 SDValue N1 = N->getOperand(1);
8620 SDValue N2 = N->getOperand(2);
8635 SDValue N1Elt = N1.getOperand(i);
8636 SDValue N2Elt = N2.getOperand(i);
8655 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
8663 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT);
8664 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT);
8668 if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8678 SDValue DAGCombiner::visitVSELECT(SDNode *N) {
8679 SDValue N0 = N->getOperand(0);
8680 SDValue N1 = N->getOperand(1);
8681 SDValue N2 = N->getOperand(2);
8685 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8689 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
8698 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
8715 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS,
8718 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
8731 if (SDValue FMinMax =
8758 SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
8759 SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
8761 SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
8783 if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
8787 if (SDValue V = foldVSelectOfConstants(N))
8793 SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
8794 SDValue N0 = N->getOperand(0);
8795 SDValue N1 = N->getOperand(1);
8796 SDValue N2 = N->getOperand(2);
8797 SDValue N3 = N->getOperand(3);
8798 SDValue N4 = N->getOperand(4);
8806 if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
8821 SDValue SelectOp = DAG.getNode(
8837 SDValue DAGCombiner::visitSETCC(SDNode *N) {
8844 SDValue Combined = SimplifySetCC(
8854 SDValue NewSetCC = rebuildSetCC(Combined);
8867 SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
8868 SDValue LHS = N->getOperand(0);
8869 SDValue RHS = N->getOperand(1);
8870 SDValue Carry = N->getOperand(2);
8871 SDValue Cond = N->getOperand(3);
8886 static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
8889 SDValue N0 = N->getOperand(0);
8908 SDValue Op1 = N0->getOperand(1);
8909 SDValue Op2 = N0->getOperand(2);
8941 SmallVector<SDValue, 8> Elts;
8950 SDValue Op = N0.getOperand(i);
8973 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
8995 SDValue UseOp = User->getOperand(i);
9034 SDValue OrigLoad, SDValue ExtLoad,
9034 SDValue OrigLoad, SDValue ExtLoad,
9039 SmallVector<SDValue, 4> Ops;
9042 SDValue SOp = SetCC->getOperand(j);
9055 SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
9056 SDValue N0 = N->getOperand(0);
9117 SmallVector<SDValue, 4> Loads;
9118 SmallVector<SDValue, 4> Chains;
9120 SDValue BasePtr = LN0->getBasePtr();
9125 SDValue SplitLoad = DAG.getExtLoad(
9137 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
9138 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
9147 SDValue Trunc =
9156 SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
9164 SDValue N0 = N->getOperand(0);
9172 SDValue N1 = N0->getOperand(0);
9202 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
9207 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
9213 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
9221 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
9236 SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
9250 SDValue VSel = Cast->getOperand(0);
9256 SDValue SetCC = VSel.getOperand(0);
9262 SDValue A = VSel.getOperand(1);
9263 SDValue B = VSel.getOperand(2);
9264 SDValue CastA, CastB;
9279 static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
9282 SDValue N0, ISD::LoadExtType ExtLoadType) {
9297 SDValue ExtLoad =
9310 static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
9312 bool LegalOperations, SDNode *N, SDValue N0,
9332 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
9343 SDValue Trunc =
9350 static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG,
9352 SDNode *N, SDValue N0,
9369 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru());
9370 SDValue NewLoad = DAG.getMaskedLoad(VT, dl, Ld->getChain(),
9379 static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
9384 SDValue SetCC = N->getOperand(0);
9389 SDValue X = SetCC.getOperand(0);
9390 SDValue Ones = SetCC.getOperand(1);
9402 SDValue NotX = DAG.getNOT(DL, X, VT);
9403 SDValue ShiftAmount = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
9410 SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
9411 SDValue N0 = N->getOperand(0);
9415 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9426 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
9438 SDValue Op = N0.getOperand(0);
9474 if (SDValue foldedExt =
9479 if (SDValue foldedExt =
9486 if (SDValue ExtLoad = CombineExtLoad(N))
9490 if (SDValue foldedExt = tryToFoldExtOfExtload(
9509 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
9515 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
9523 SDValue TruncAnd =
9530 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
9539 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
9543 SDValue N00 = N0.getOperand(0);
9544 SDValue N01 = N0.getOperand(1);
9573 SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
9591 SDValue ExtTrueVal = (SetCCWidth == 1)
9594 SDValue Zero = DAG.getConstant(0, DL, VT);
9595 if (SDValue SCC =
9607 SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
9618 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
9627 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT);
9636 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT);
9647 static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
9647 static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
9660 SDValue Op0 = N->getOperand(0);
9661 SDValue Op1 = N->getOperand(1);
9679 static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) {
9683 SDValue CtPop = Extend->getOperand(0);
9695 SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT);
9699 SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
9700 SDValue N0 = N->getOperand(0);
9703 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9715 SDValue Op;
9733 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
9751 SDValue Op = N0.getOperand(0);
9754 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
9762 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
9764 SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
9780 SDValue X = N0.getOperand(0).getOperand(0);
9790 if (SDValue foldedExt =
9795 if (SDValue foldedExt =
9802 if (SDValue ExtLoad = CombineExtLoad(N))
9833 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
9840 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
9848 SDValue TruncAnd =
9855 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
9866 if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
9870 if (SDValue foldedExt = tryToFoldExtOfExtload(
9874 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
9891 SDValue VecOnes = DAG.getConstant(1, DL, VT);
9894 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
9903 SDValue VsetCC =
9912 if (SDValue SCC = SimplifySelectCC(
9924 SDValue ShAmt = N0.getOperand(1);
9926 SDValue InnerZExt = N0.getOperand(0);
9946 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
9949 if (SDValue NewCtPop = widenCtPop(N, DAG))
9955 SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
9956 SDValue N0 = N->getOperand(0);
9959 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9973 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
9996 SDValue X = N0.getOperand(0).getOperand(0);
10018 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
10030 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
10047 SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
10082 SDValue VsetCC =
10091 if (SDValue SCC = SimplifySelectCC(
10098 if (SDValue NewCtPop = widenCtPop(N, DAG))
10104 SDValue DAGCombiner::visitAssertExt(SDNode *N) {
10106 SDValue N0 = N->getOperand(0);
10107 SDValue N1 = N->getOperand(1);
10122 SDValue BigA = N0.getOperand(0);
10130 SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
10131 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10142 SDValue BigA = N0.getOperand(0);
10150 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10165 SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
10169 SDValue N0 = N->getOperand(0);
10226 SDValue SRL = N0;
10314 SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
10320 SDValue Load;
10336 SDValue Result = Load;
10362 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
10371 SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
10372 SDValue N0 = N->getOperand(0);
10373 SDValue N1 = N->getOperand(1);
10401 SDValue N00 = N0.getOperand(0);
10423 SDValue N00 = N0.getOperand(0);
10440 if (SDValue NarrowLoad = ReduceLoadWidth(N))
10469 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
10485 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
10496 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
10505 SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
10506 SDValue N0 = N->getOperand(0);
10512 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10521 SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
10522 SDValue N0 = N->getOperand(0);
10528 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10537 SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
10538 SDValue N0 = N->getOperand(0);
10553 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
10599 SDValue EltNo = N0->getOperand(1);
10617 SDValue Cond = N0.getOperand(0);
10618 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
10619 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
10628 SDValue Amt = N0.getOperand(1);
10635 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
10649 SmallVector<SDValue, 8> TruncOps;
10650 for (const SDValue &Op : N0->op_values()) {
10651 SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op);
10665 SDValue BuildVect = N0.getOperand(0);
10679 SmallVector<SDValue, 8> Opnds;
10695 if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
10702 if (SDValue Reduced = ReduceLoadWidth(N))
10711 SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
10725 SDValue V;
10730 SDValue X = N0.getOperand(i);
10749 SmallVector<SDValue, 8> Opnds;
10755 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
10768 SDValue VecSrc = N0.getOperand(0);
10796 auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
10797 auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
10807 SDValue N00 = N0.getOperand(0);
10818 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
10840 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
10841 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
10851 SDValue Elt = N->getOperand(i);
10859 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
10897 static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
10907 SDValue N0 = N->getOperand(0);
10935 SDValue LogicOp0 = N0.getOperand(0);
10940 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0));
10950 SDValue DAGCombiner::visitBITCAST(SDNode *N) {
10951 SDValue N0 = N->getOperand(0);
10982 SDValue C = DAG.getBitcast(VT, N0);
11009 SDValue Load =
11018 if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
11037 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
11043 SDValue SignBit = DAG.getConstant(
11045 SDValue FlipBit;
11051 SDValue Hi =
11059 SDValue FlipBits =
11090 SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
11113 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11115 SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
11117 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
11119 SDValue XorResult64 = DAG.getNode(
11124 SDValue FlipBit =
11128 SDValue FlipBits =
11138 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11149 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT))
11177 SDValue SV0 = PeekThroughBitcast(N0->getOperand(0));
11178 SDValue SV1 = PeekThroughBitcast(N0->getOperand(1));
11189 SDValue LegalShuffle =
11198 SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
11205 SDValue DAGCombiner::
11218 SmallVector<SDValue, 8> Ops;
11219 for (SDValue Op : BV->op_values()) {
11261 SmallVector<SDValue, 8> Ops;
11270 SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
11293 SmallVector<SDValue, 8> Ops;
11295 for (const SDValue &Op : BV->op_values()) {
11324 SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
11325 SDValue N0 = N->getOperand(0);
11326 SDValue N1 = N->getOperand(1);
11391 SDValue N00 = N0.getOperand(0);
11405 SDValue N10 = N1.getOperand(0);
11457 SDValue N02 = N0.getOperand(2);
11459 SDValue N020 = N02.getOperand(0);
11486 SDValue N00 = N0.getOperand(0);
11488 SDValue N002 = N00.getOperand(2);
11501 SDValue N12 = N1.getOperand(2);
11503 SDValue N120 = N12.getOperand(0);
11519 SDValue N10 = N1.getOperand(0);
11521 SDValue N102 = N10.getOperand(2);
11536 SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
11537 SDValue N0 = N->getOperand(0);
11538 SDValue N1 = N->getOperand(1);
11599 SDValue N00 = N0.getOperand(0).getOperand(0);
11600 SDValue N01 = N0.getOperand(0).getOperand(1);
11611 SDValue N00 = N0.getOperand(0);
11627 SDValue N10 = N1.getOperand(0);
11647 SDValue N00 = N0.getOperand(0);
11649 SDValue N000 = N00.getOperand(0);
11670 SDValue N00 = N0.getOperand(0);
11672 SDValue N000 = N00.getOperand(0);
11706 SDValue N20 = N1.getOperand(2).getOperand(0);
11707 SDValue N21 = N1.getOperand(2).getOperand(1);
11721 SDValue N02 = N0.getOperand(2);
11723 SDValue N020 = N02.getOperand(0);
11746 SDValue N00 = N0.getOperand(0);
11748 SDValue N002 = N00.getOperand(2);
11771 SDValue N120 = N1.getOperand(2).getOperand(0);
11774 SDValue N1200 = N120.getOperand(0);
11775 SDValue N1201 = N120.getOperand(1);
11797 SDValue CvtSrc = N1.getOperand(0);
11798 SDValue N100 = CvtSrc.getOperand(0);
11799 SDValue N101 = CvtSrc.getOperand(1);
11800 SDValue N102 = CvtSrc.getOperand(2);
11803 SDValue N1020 = N102.getOperand(0);
11804 SDValue N1021 = N102.getOperand(1);
11827 SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
11828 SDValue N0 = N->getOperand(0);
11829 SDValue N1 = N->getOperand(1);
11878 if (SDValue FMA = FuseFADD(N0, N1, Flags))
11880 if (SDValue FMA = FuseFADD(N1, N0, Flags))
11911 if (SDValue FMA = FuseFSUB(N0, N1, Flags))
11913 if (SDValue FMA = FuseFSUB(N1, N0, Flags))
11919 SDValue DAGCombiner::visitFADD(SDNode *N) {
11920 SDValue N0 = N->getOperand(0);
11921 SDValue N1 = N->getOperand(1);
11931 if (SDValue FoldedVOp = SimplifyVBinOp(N))
11948 if (SDValue NewSel = foldBinOpIntoSelect(N))
11974 SDValue B = N0.getOperand(0);
11975 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
11980 SDValue B = N1.getOperand(0);
11981 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
12009 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1, Flags);
12023 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12032 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12044 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12053 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12091 if (SDValue Fused = visitFADDForFMACombine(N)) {
12098 SDValue DAGCombiner::visitFSUB(SDNode *N) {
12099 SDValue N0 = N->getOperand(0);
12100 SDValue N1 = N->getOperand(1);
12110 if (SDValue FoldedVOp = SimplifyVBinOp(N))
12117 if (SDValue NewSel = foldBinOpIntoSelect(N))
12167 if (SDValue Fused = visitFSUBForFMACombine(N)) {
12177 bool DAGCombiner::isCheaperToUseNegatedFPOps(SDValue X, SDValue Y) {
12177 bool DAGCombiner::isCheaperToUseNegatedFPOps(SDValue X, SDValue Y) {
12190 SDValue DAGCombiner::visitFMUL(SDNode *N) {
12191 SDValue N0 = N->getOperand(0);
12192 SDValue N1 = N->getOperand(1);
12203 if (SDValue FoldedVOp = SimplifyVBinOp(N))
12216 if (SDValue NewSel = foldBinOpIntoSelect(N))
12230 SDValue N00 = N0.getOperand(0);
12231 SDValue N01 = N0.getOperand(1);
12236 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1, Flags);
12245 const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
12246 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1, Flags);
12262 SDValue NegN0 =
12264 SDValue NegN1 =
12274 SDValue Select = N0, X = N1;
12278 SDValue Cond = Select.getOperand(0);
12316 if (SDValue Fused = visitFMULForFMADistributiveCombine(N)) {
12324 SDValue DAGCombiner::visitFMA(SDNode *N) {
12325 SDValue N0 = N->getOperand(0);
12326 SDValue N1 = N->getOperand(1);
12327 SDValue N2 = N->getOperand(2);
12347 SDValue NegN0 =
12349 SDValue NegN1 =
12402 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0);
12446 SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
12456 SDValue N0 = N->getOperand(0);
12463 SDValue N1 = N->getOperand(1);
12494 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
12495 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags);
12499 SDValue Dividend = U->getOperand(0);
12501 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
12513 SDValue DAGCombiner::visitFDIV(SDNode *N) {
12514 SDValue N0 = N->getOperand(0);
12515 SDValue N1 = N->getOperand(1);
12525 if (SDValue FoldedVOp = SimplifyVBinOp(N))
12532 if (SDValue NewSel = foldBinOpIntoSelect(N))
12535 if (SDValue V = combineRepeatedFPDivisors(N))
12561 if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0), Flags))
12565 if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0),
12573 if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0),
12582 SDValue SqrtOp;
12583 SDValue OtherOp;
12594 if (SDValue RV = buildRsqrtEstimate(SqrtOp.getOperand(0), Flags)) {
12603 if (SDValue RV = BuildDivEstimate(N0, N1, Flags))
12617 SDValue DAGCombiner::visitFREM(SDNode *N) {
12618 SDValue N0 = N->getOperand(0);
12619 SDValue N1 = N->getOperand(1);
12628 if (SDValue NewSel = foldBinOpIntoSelect(N))
12634 SDValue DAGCombiner::visitFSQRT(SDNode *N) {
12640 SDValue N0 = N->getOperand(0);
12651 SDValue N1 = N->getOperand(1);
12665 SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
12666 SDValue N0 = N->getOperand(0);
12667 SDValue N1 = N->getOperand(1);
12712 SDValue DAGCombiner::visitFPOW(SDNode *N) {
12777 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags);
12778 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags);
12788 static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG,
12811 SDValue N0 = N->getOperand(0);
12823 SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
12824 SDValue N0 = N->getOperand(0);
12856 SDValue Ops[] =
12870 SDValue Ops[] =
12878 if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
12884 SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
12885 SDValue N0 = N->getOperand(0);
12916 SDValue Ops[] =
12924 if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
12931 static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG) {
12932 SDValue N0 = N->getOperand(0);
12938 SDValue Src = N0.getOperand(0);
12972 SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
12973 SDValue N0 = N->getOperand(0);
12987 SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
12988 SDValue N0 = N->getOperand(0);
13002 SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
13003 SDValue N0 = N->getOperand(0);
13004 SDValue N1 = N->getOperand(1);
13045 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT,
13052 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
13058 SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
13059 SDValue N0 = N->getOperand(0);
13080 SDValue In = N0.getOperand(0);
13092 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
13105 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
13111 SDValue DAGCombiner::visitFCEIL(SDNode *N) {
13112 SDValue N0 = N->getOperand(0);
13122 SDValue DAGCombiner::visitFTRUNC(SDNode *N) {
13123 SDValue N0 = N->getOperand(0);
13146 SDValue DAGCombiner::visitFFLOOR(SDNode *N) {
13147 SDValue N0 = N->getOperand(0);
13158 SDValue DAGCombiner::visitFNEG(SDNode *N) {
13159 SDValue N0 = N->getOperand(0);
13174 SDValue Int = N0.getOperand(0);
13215 static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
13217 SDValue N0 = N->getOperand(0);
13218 SDValue N1 = N->getOperand(1);
13237 SDValue DAGCombiner::visitFMINNUM(SDNode *N) {
13241 SDValue DAGCombiner::visitFMAXNUM(SDNode *N) {
13245 SDValue DAGCombiner::visitFMINIMUM(SDNode *N) {
13249 SDValue DAGCombiner::visitFMAXIMUM(SDNode *N) {
13253 SDValue DAGCombiner::visitFABS(SDNode *N) {
13254 SDValue N0 = N->getOperand(0);
13272 SDValue Int = N0.getOperand(0);
13296 SDValue DAGCombiner::visitBRCOND(SDNode *N) {
13297 SDValue Chain = N->getOperand(0);
13298 SDValue N1 = N->getOperand(1);
13299 SDValue N2 = N->getOperand(2);
13318 if (SDValue NewN1 = rebuildSetCC(N1))
13325 SDValue DAGCombiner::rebuildSetCC(SDValue N) {
13325 SDValue DAGCombiner::rebuildSetCC(SDValue N) {
13351 SDValue Op0 = N.getOperand(0);
13352 SDValue Op1 = N.getOperand(1);
13355 SDValue AndOp1 = Op0.getOperand(1);
13381 SDValue Tmp = visitXOR(N.getNode());
13398 SDValue Op0 = TheXor->getOperand(0);
13399 SDValue Op1 = TheXor->getOperand(1);
13423 SDValue DAGCombiner::visitBR_CC(SDNode *N) {
13425 SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3);
13434 SDValue Simp = SimplifySetCC(getSetCCResultType(CondLHS.getValueType()),
13506 SDValue Ptr;
13536 SDValue BasePtr;
13537 SDValue Offset;
13570 SDValue Val = cast<StoreSDNode>(N)->getValue();
13609 SDValue Op1 = Use.getUser()->getOperand((UI.getOperandNo() + 1) & 1);
13645 SDValue Result;
13710 SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0));
13711 SDValue NewOp2 = Result.getValue(isLoad ? 1 : 0);
13713 SDValue NewUse = DAG.getNode(Opcode,
13737 SDValue Ptr;
13768 SDValue BasePtr;
13769 SDValue Offset;
13820 SDValue Result = isLoad
13854 SDValue DAGCombiner::SplitIndexingFromLoad(LoadSDNode *LD) {
13857 SDValue BP = LD->getOperand(1);
13858 SDValue Inc = LD->getOperand(2);
13881 bool DAGCombiner::getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val) {
13906 bool DAGCombiner::extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val) {
13932 SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
13935 SDValue Chain = LD->getOperand(0);
13970 SDValue Idx = DAG.getNode(Opc, SDLoc(LD), LD->getOperand(1).getValueType(),
13972 SDValue Ops[] = {Val, Idx, Chain};
13990 auto Mask =
13994 auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask);
14003 SDValue Val;
14029 SDValue DAGCombiner::visitLOAD(SDNode *N) {
14031 SDValue Chain = LD->getChain();
14032 SDValue Ptr = LD->getBasePtr();
14072 SDValue Undef = DAG.getUNDEF(N->getValueType(0));
14073 SDValue Index;
14096 if (auto V = ForwardStoreValueToDirectLoad(LD))
14103 SDValue NewLoad = DAG.getExtLoad(
14116 SDValue BetterChain = FindBetterChain(LD, Chain);
14120 SDValue ReplLoad;
14134 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N),
14381 SDValue loadSlice() const {
14383 const SDValue &OldBaseAddr = Origin->getBasePtr();
14384 SDValue BaseAddr = OldBaseAddr;
14400 SDValue LastInst =
14679 SmallVector<SDValue, 8> ArgChains;
14684 SDValue SliceInst = LSIt->loadSlice();
14693 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other,
14704 CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
14704 CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
14704 CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
14773 static SDValue
14775 SDValue IVal, StoreSDNode *St,
14817 SDValue Ptr = St->getBasePtr();
14838 SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
14843 SDValue Chain = ST->getChain();
14844 SDValue Value = ST->getValue();
14845 SDValue Ptr = ST->getBasePtr();
14862 if (SDValue NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
14869 if (SDValue NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
14878 SDValue N0 = Value.getOperand(0);
14888 SDValue N1 = Value.getOperand(1);
14932 SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LD),
14936 SDValue NewLD =
14940 SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
14943 SDValue NewST =
14963 SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
14965 SDValue Value = ST->getValue();
14992 SDValue NewLD =
14996 SDValue NewST =
15025 SDValue &AddNode,
15026 SDValue &ConstNode) {
15085 SDValue DAGCombiner::getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
15087 SmallVector<SDValue, 8> Chains;
15127 SDValue StoredVal;
15130 SmallVector<SDValue, 8> BuildVector;
15133 SDValue Val = St->getValue();
15159 SmallVector<SDValue, 8> Ops;
15162 SDValue Val = peekThroughBitcasts(St->getValue());
15179 SDValue Vec = Val.getOperand(0);
15180 SDValue Idx = Val.getOperand(1);
15206 SDValue Val = St->getValue();
15231 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
15234 SDValue NewStore;
15244 SDValue ExtendedStoreVal =
15270 SDValue Val = peekThroughBitcasts(St->getValue());
15310 SDValue OtherBC = peekThroughBitcasts(Other->getValue());
15442 for (SDValue Op : N->ops())
15508 SDValue StoredVal = peekThroughBitcasts(St->getValue());
15595 SDValue StoredVal = ST->getValue();
15780 SDValue Val = peekThroughBitcasts(St->getValue());
15828 SDValue FirstChain = FirstLoad->getChain();
15958 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
15971 SDValue NewLoad, NewStore;
16004 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
16019 SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
16019 SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
16021 SDValue ReplStore;
16034 SDValue Token = DAG.getNode(ISD::TokenFactor, SL,
16044 SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) {
16045 SDValue Value = ST->getValue();
16051 SDValue Chain = ST->getChain();
16052 SDValue Ptr = ST->getBasePtr();
16061 SDValue Tmp;
16098 SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, SDLoc(CFP), MVT::i32);
16099 SDValue Hi = DAG.getConstant(Val >> 32, SDLoc(CFP), MVT::i32);
16107 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16112 SDValue St1 = DAG.getStore(Chain, DL, Hi, Ptr,
16123 SDValue DAGCombiner::visitSTORE(SDNode *N) {
16125 SDValue Chain = ST->getChain();
16126 SDValue Value = ST->getValue();
16127 SDValue Ptr = ST->getBasePtr();
16158 SDValue NewStore =
16171 if (SDValue NewST = TransformFPLoadStorePair(N))
16175 if (SDValue Store = MatchStoreCombine(ST))
16202 if (SDValue Shorter = DAG.GetDemandedBits(Value, TruncDemandedBits))
16266 SDValue ChainValue = ST1->getValue();
16274 SDValue NewSDVal =
16326 if (SDValue NewSt = replaceStoreOfFPConstant(ST))
16330 if (SDValue NewSt = splitMergedValStore(ST))
16336 SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) {
16345 SmallVector<SDValue, 8> Chains = {N->getOperand(0)};
16347 SDValue Chain = Chains.back();
16413 SDValue DAGCombiner::splitMergedValStore(StoreSDNode *ST) {
16422 SDValue Val = ST->getValue();
16430 SDValue Op1 = Val.getOperand(0);
16431 SDValue Op2 = Val.getOperand(1);
16432 SDValue Lo, Hi;
16480 SDValue Chain = ST->getChain();
16481 SDValue Ptr = ST->getBasePtr();
16483 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16489 SDValue St1 =
16497 SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) {
16498 SDValue InsertVal = N->getOperand(1);
16499 SDValue Vec = N->getOperand(0);
16509 SDValue X = Vec.getOperand(0);
16510 SDValue Y = Vec.getOperand(1);
16531 SDValue LegalShuffle =
16547 SDValue SubVec = InsertVal.getOperand(0);
16548 SDValue DestVec = N->getOperand(0);
16577 SmallVector<SDValue, 8> ConcatOps(ExtendRatio, DAG.getUNDEF(SubVecVT));
16579 SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps);
16582 SDValue DestVecBC = DAG.getBitcast(ShufVT, DestVec);
16583 SDValue Shuf = DAG.getVectorShuffle(ShufVT, DL, DestVecBC, PaddedSubV, Mask);
16590 SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
16591 SDValue InVec = N->getOperand(0);
16592 SDValue InVal = N->getOperand(1);
16593 SDValue EltNo = N->getOperand(2);
16610 SmallVector<SDValue, 8> Ops(NumElts, InVal);
16618 if (SDValue Shuf = combineInsertEltToShuffle(N, Elt))
16633 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16648 SmallVector<SDValue, 8> Ops;
16673 SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
16674 SDValue EltNo,
16694 SDValue NewPtr = OriginalLoad->getBasePtr();
16695 SDValue Offset;
16721 SDValue Load;
16722 SDValue Chain;
16746 SDValue From[] = { SDValue(EVE, 0), SDValue(OriginalLoad, 1) };
16747 SDValue To[] = { Load, Chain };
16761 static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
16764 SDValue Vec = ExtElt->getOperand(0);
16765 SDValue Index = ExtElt->getOperand(1);
16778 SDValue Op0 = Vec.getOperand(0);
16779 SDValue Op1 = Vec.getOperand(1);
16786 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
16787 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
16794 SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
16795 SDValue VecOp = N->getOperand(0);
16796 SDValue Index = N->getOperand(1);
16809 SDValue Elt = VecOp.getOperand(1);
16818 SDValue InOp = VecOp.getOperand(0);
16836 SDValue Elt = VecOp.getOperand(IndexC->getZExtValue());
16857 SDValue BCSrc = VecOp.getOperand(0);
16865 SDValue X = BCSrc.getOperand(0);
16884 if (SDValue BO = scalarizeExtractedBinop(N, DAG, LegalOperations))
16903 SDValue SVInVec;
16912 SDValue InOp = SVInVec.getOperand(OrigElt);
17054 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
17077 SDValue In = N->getOperand(i);
17123 SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
17127 SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
17131 SDValue Cast = N->getOperand(i);
17135 SDValue In;
17158 SDValue BV = DAG.getBuildVector(VecVT, DL, Ops);
17166 SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N,
17168 SDValue VecIn1, SDValue VecIn2,
17168 SDValue VecIn1, SDValue VecIn2,
17171 SDValue ZeroIdx = DAG.getConstant(0, DL, IdxTy);
17192 SmallVector<SDValue, 2> ConcatOps(NumConcats, DAG.getUNDEF(InVT1));
17237 SmallVector<SDValue, 2> ConcatOps(2, DAG.getUNDEF(InVT2));
17275 SDValue Shuffle = DAG.getVectorShuffle(InVT1, DL, VecIn1, VecIn2, Mask);
17282 static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) {
17290 SDValue Op = BV->getOperand(i);
17309 SDValue Zext = BV->getOperand(ZextElt);
17318 SDValue Extract = Zext.getOperand(0);
17350 SDValue ZeroVec = DAG.getConstant(0, DL, VecVT);
17352 SDValue Shuf = TLI.buildLegalVectorShuffle(VecVT, DL, Extract.getOperand(0),
17362 SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
17370 if (SDValue V = reduceBuildVecToShuffleWithZero(N, DAG))
17387 SmallVector<SDValue, 8> VecIn;
17391 SDValue Op = N->getOperand(i);
17410 SDValue ExtractedFromVec = Op.getOperand(0);
17444 SDValue Vec = VecIn.back();
17464 SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
17466 SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
17501 SmallVector<SDValue, 4> Shuffles;
17504 SDValue VecLeft = VecIn[LeftIdx];
17505 SDValue VecRight =
17508 if (SDValue Shuffle = createBuildVecShuffle(DL, N, VectorMask, VecLeft,
17577 SDValue DAGCombiner::convertBuildVecZextToZext(SDNode *N) {
17584 SDValue Op0 = N->getOperand(0);
17603 SDValue In = Op0.getOperand(0).getOperand(0);
17624 SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
17640 if (SDValue Splat = cast<BuildVectorSDNode>(N)->getSplatValue()) {
17648 SmallVector<SDValue, 8> Ops(N->getNumOperands(), Splat);
17649 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N),
17659 if (SDValue V = cast<BuildVectorSDNode>(N)->getSplatValue()) {
17666 SDValue Op0 = N->getOperand(0);
17693 if (SDValue V = convertBuildVecZextToZext(N))
17696 if (SDValue V = reduceBuildVecExtToExtBuildVec(N))
17699 if (SDValue V = reduceBuildVecToShuffle(N))
17705 static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) {
17715 SmallVector<SDValue, 8> Ops;
17718 SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT);
17723 for (const SDValue &Op : N->ops()) {
17750 for (SDValue &Op : Ops) {
17770 static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG) {
17776 SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT);
17779 for (SDValue Op : N->ops()) {
17792 SDValue ExtVec = Op.getOperand(0);
17842 SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
17856 SDValue In = N->getOperand(0);
17863 SmallVector<SDValue, 4> Ops(In->op_begin(), In->op_end());
17868 SDValue Scalar = peekThroughOneUseBitcasts(In);
17906 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar);
17919 SmallVector<SDValue, 8> Opnds;
17927 for (const SDValue &Op : N->ops())
17936 for (const SDValue &Op : N->ops()) {
17961 if (SDValue V = combineConcatVectorOfScalars(N, DAG))
17966 if (SDValue V = combineConcatVectorOfExtracts(N, DAG))
17973 SDValue SingleSource = SDValue();
17977 SDValue Op = N->getOperand(i);
18019 static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT) {
18019 static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT) {
18019 static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT) {
18034 static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract,
18037 SDValue BinOp = Extract->getOperand(0);
18043 SDValue Bop0 = BinOp.getOperand(0), Bop1 = BinOp.getOperand(1);
18047 SDValue Index = Extract->getOperand(1);
18052 SDValue Sub0 = getSubVectorSrc(Bop0, Index, SubVT);
18053 SDValue Sub1 = getSubVectorSrc(Bop1, Index, SubVT);
18070 static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) {
18074 if (SDValue V = narrowInsertExtractVectorBinOp(Extract, DAG))
18085 SDValue BinOp = peekThroughBitcasts(Extract->getOperand(0));
18130 SDValue NewExtIndex = DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT);
18131 SDValue X = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
18133 SDValue Y = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
18135 SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y,
18160 SDValue SubVecL = GetSubVector(peekThroughBitcasts(BinOp.getOperand(0)));
18161 SDValue SubVecR = GetSubVector(peekThroughBitcasts(BinOp.getOperand(1)));
18170 SDValue IndexC = DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT);
18171 SDValue X = SubVecL ? DAG.getBitcast(NarrowBVT, SubVecL)
18175 SDValue Y = SubVecR ? DAG.getBitcast(NarrowBVT, SubVecR)
18179 SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y);
18189 static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
18209 SDValue BaseAddr = Ld->getOperand(1);
18213 SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
18217 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO);
18222 SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
18224 SDValue V = N->getOperand(0);
18231 if (SDValue NarrowLoad = narrowExtractedVectorLoad(N, DAG))
18236 SDValue Index = N->getOperand(1);
18251 SDValue SrcOp = V.getOperand(0);
18263 SDValue NewIndex = DAG.getIntPtrConstant(IndexValScaled, DL);
18264 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT,
18311 SDValue Src = V->getOperand(IdxVal);
18318 SDValue BuildVec = DAG.getBuildVector(
18352 if (SDValue NarrowBOp = narrowExtractedVectorBinOp(N, DAG))
18365 static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf,
18367 SDValue N0 = Shuf->getOperand(0), N1 = Shuf->getOperand(1);
18401 SDValue X = N0.getOperand(0), Y = N1.getOperand(0);
18403 SDValue Shuf0 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask0);
18404 SDValue Shuf1 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask1);
18410 static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) {
18414 SDValue N0 = N->getOperand(0);
18415 SDValue N1 = N->getOperand(1);
18419 SmallVector<SDValue, 4> Ops;
18489 static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN,
18494 SDValue N0 = SVN->getOperand(0);
18495 SDValue N1 = SVN->getOperand(1);
18520 if (SDValue Splat0 = BV0->getSplatValue())
18523 SmallVector<SDValue, 8> Ops;
18524 SmallSet<SDValue, 16> DuplicateOps;
18526 SDValue Op = DAG.getUNDEF(VT.getScalarType());
18529 SDValue &S = (M < (int)NumElts ? N0 : N1);
18533 SDValue Op0 = S.getOperand(0);
18556 for (SDValue &Op : Ops)
18559 for (SDValue &Op : Ops)
18570 static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN,
18584 SDValue N0 = SVN->getOperand(0);
18627 static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN,
18636 SDValue N0 = peekThroughBitcasts(SVN->getOperand(0));
18644 SDValue N00 = N0.getOperand(0);
18688 static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf,
18766 static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf,
18772 SDValue Op0 = Shuf->getOperand(0);
18773 SDValue Op1 = Shuf->getOperand(1);
18810 SDValue NewInsIndex = DAG.getConstant(ShufOp0Index, SDLoc(Shuf),
18820 static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf) {
18844 SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
18848 SDValue N0 = N->getOperand(0);
18849 SDValue N1 = N->getOperand(1);
18890 if (SDValue InsElt = replaceShuffleOfInsert(SVN, DAG))
18894 if (SDValue V = combineShuffleOfSplatVal(SVN, DAG))
18905 SDValue L = N0.getOperand(0), R = N0.getOperand(1);
18908 SDValue Index = DAG.getIntPtrConstant(SplatIndex, DL);
18909 SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index);
18910 SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index);
18911 SDValue NewBO = DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR,
18913 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO);
18923 SDValue ConvInput = V->getOperand(0);
18932 SDValue Base;
18954 SDValue Splatted = V->getOperand(SplatIndex);
18955 SmallVector<SDValue, 8> Ops(NumElts, Splatted);
18956 SDValue NewBV = DAG.getBuildVector(V->getValueType(0), SDLoc(N), Ops);
18972 if (SDValue ShufOp = simplifyShuffleOfShuffle(SVN))
18976 if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations))
18980 if (SDValue V = combineTruncationShuffle(SVN, DAG))
18988 if (SDValue V = partitionShuffleOfConcats(N, DAG))
18995 if (SDValue Res = combineShuffleOfScalars(SVN, DAG, TLI))
19015 SDValue BC0 = peekThroughOneUseBitcasts(N0);
19044 SDValue SV0 = BC0->getOperand(0);
19045 SDValue SV1 = BC0->getOperand(1);
19075 SDValue SV0 = N1->getOperand(0);
19076 SDValue SV1 = N1->getOperand(1);
19105 SDValue SV0, SV1;
19117 SDValue CurrentVec;
19185 if (SDValue V = foldShuffleOfConcatUndefs(SVN, DAG))
19191 SDValue DAGCombiner::visitSCALAR_TO_VECTOR(SDNode *N) {
19192 SDValue InVal = N->getOperand(0);
19198 SDValue InVec = InVal->getOperand(0);
19199 SDValue EltNo = InVal->getOperand(1);
19210 SDValue Val =
19216 SDValue LegalShuffle =
19227 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(N), IdxTy);
19242 SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
19244 SDValue N0 = N->getOperand(0);
19245 SDValue N1 = N->getOperand(1);
19246 SDValue N2 = N->getOperand(2);
19277 SDValue CN0 = N0.getOperand(0);
19278 SDValue CN1 = N1.getOperand(0);
19284 SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N),
19317 SDValue N0Src = peekThroughBitcasts(N0);
19318 SDValue N1Src = peekThroughBitcasts(N1);
19325 SDValue NewIdx;
19342 SDValue Res = DAG.getBitcast(NewVT, N0Src);
19359 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT,
19373 SmallVector<SDValue, 8> Ops(N0->op_begin(), N0->op_end());
19386 SDValue DAGCombiner::visitFP_TO_FP16(SDNode *N) {
19387 SDValue N0 = N->getOperand(0);
19396 SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) {
19397 SDValue N0 = N->getOperand(0);
19411 SDValue DAGCombiner::visitVECREDUCE(SDNode *N) {
19412 SDValue N0 = N->getOperand(0);
19419 SDValue Res = DAG.getNode(
19445 SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
19449 SDValue LHS = N->getOperand(0);
19450 SDValue RHS = peekThroughBitcasts(N->getOperand(1));
19475 SDValue Elt = RHS.getOperand(EltIdx);
19513 SDValue Zero = DAG.getConstant(0, DL, ClearVT);
19526 if (SDValue S = BuildClearMask(Split))
19534 static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG) {
19535 SDValue N0 = N->getOperand(0);
19536 SDValue N1 = N->getOperand(1);
19546 SDValue Src0 = DAG.getSplatSourceVector(N0, Index0);
19547 SDValue Src1 = DAG.getSplatSourceVector(N1, Index1);
19556 SDValue IndexC =
19558 SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, N0, IndexC);
19559 SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, N1, IndexC);
19560 SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, X, Y, N->getFlags());
19569 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT));
19575 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO);
19580 SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
19584 SDValue LHS = N->getOperand(0);
19585 SDValue RHS = N->getOperand(1);
19586 SDValue Ops[] = {LHS, RHS};
19591 if (SDValue Fold = DAG.FoldConstantVectorArithmetic(
19611 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0),
19613 SDValue UndefV = LHS.getOperand(1);
19626 SDValue X = LHS.getOperand(1);
19627 SDValue Y = RHS.getOperand(1);
19628 SDValue Z = LHS.getOperand(2);
19634 SDValue VecC =
19636 SDValue NarrowBO = DAG.getNode(Opcode, DL, NarrowVT, X, Y);
19663 SmallVector<SDValue, 4> ConcatOps;
19674 if (SDValue V = scalarizeBinOpOfSplats(N, DAG))
19680 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
19680 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
19680 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
19681 SDValue N2) {
19684 SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2,
19695 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0),
19700 SDValue SelectNode = DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC,
19716 bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
19717 SDValue RHS) {
19723 SDValue Sqrt = RHS;
19725 SDValue CmpLHS;
19734 SDValue Cmp = TheSelect->getOperand(0);
19820 SDValue Addr;
19866 SDValue Load;
19905 SDValue DAGCombiner::foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0,
19905 SDValue DAGCombiner::foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0,
19906 SDValue N1, SDValue N2, SDValue N3,
19906 SDValue N1, SDValue N2, SDValue N3,
19906 SDValue N1, SDValue N2, SDValue N3,
19940 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, ShiftAmtTy);
19941 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt);
19955 SDValue ShiftAmt = DAG.getConstant(XType.getSizeInBits() - 1, DL, ShiftAmtTy);
19956 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt);
19974 SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
19975 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
19975 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
19975 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
19975 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
20006 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(DAG.getDataLayout()),
20012 SDValue Zero = DAG.getIntPtrConstant(0, DL);
20014 SDValue One = DAG.getIntPtrConstant(EltSize, SDLoc(FV));
20015 SDValue Cond =
20018 SDValue CstOffset = DAG.getSelect(DL, Zero.getValueType(), Cond, One, Zero);
20029 SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
20029 SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
20029 SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
20030 SDValue N2, SDValue N3, ISD::CondCode CC,
20030 SDValue N2, SDValue N3, ISD::CondCode CC,
20043 if (SDValue SCC = DAG.FoldSetCC(CmpResVT, N0, N1, CC, DL)) {
20052 if (SDValue V =
20056 if (SDValue V = foldSelectCCToShiftAnd(DL, N0, N1, N2, N3, CC))
20072 SDValue AndLHS = N0->getOperand(0);
20077 SDValue ShlAmt =
20080 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt);
20084 SDValue ShrAmt =
20087 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt);
20112 SDValue Temp, SCC;
20147 SDValue ValueOnZero = N2;
20148 SDValue Count = N3;
20177 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
20177 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
20177 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
20189 SDValue DAGCombiner::BuildSDIV(SDNode *N) {
20196 if (SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, Built)) {
20207 SDValue DAGCombiner::BuildSDIVPow2(SDNode *N) {
20217 if (SDValue S = TLI.BuildSDIVPow2(N, C->getAPIntValue(), DAG, Built)) {
20230 SDValue DAGCombiner::BuildUDIV(SDNode *N) {
20237 if (SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, Built)) {
20248 SDValue DAGCombiner::BuildLogBase2(SDValue V, const SDLoc &DL) {
20248 SDValue DAGCombiner::BuildLogBase2(SDValue V, const SDLoc &DL) {
20251 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V);
20252 SDValue Base = DAG.getConstant(EltBits - 1, DL, VT);
20253 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz);
20265 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
20265 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
20265 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
20284 if (SDValue Est = TLI.getRecipEstimate(Op, DAG, Enabled, Iterations)) {
20289 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
20294 SDValue MulEst = Est;
20301 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags);
20332 SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est,
20332 SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est,
20332 SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est,
20337 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT);
20341 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags);
20346 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags);
20364 SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est,
20364 SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est,
20364 SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est,
20369 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT);
20370 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT);
20379 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags);
20380 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags);
20381 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags);
20386 SDValue LHS;
20404 SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
20404 SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
20425 if (SDValue Est =
20447 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
20448 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
20449 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
20450 SDValue IsDenorm = DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
20454 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
20455 SDValue IsZero = DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
20466 SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags) {
20466 SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags) {
20470 SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) {
20470 SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) {
20480 SDValue BasePtr;
20604 void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
20605 SmallVectorImpl<SDValue> &Aliases) {
20606 SmallVector<SDValue, 8> Chains; // List of chains to visit.
20618 std::function<bool(SDValue &)> ImproveChain = [&](SDValue &C) -> bool {
20664 SDValue Chain = Chains.pop_back_val();
20711 SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
20711 SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
20716 SmallVector<SDValue, 8> Aliases;
20816 SDValue NewChain = STChain->getChain();
20817 SmallVector<SDValue, 8> TFOps;
20820 SDValue BetterChain = FindBetterChain(S, NewChain);
20828 SDValue BetterChain = FindBetterChain(St, NewChain);
20829 SDValue NewST;
20850 SDValue TF = DAG.getTokenFactor(SDLoc(STChain), TFOps);
20855 for (const SDValue &Op : TF->ops())
20880 SDValue BetterChain = FindBetterChain(St, St->getChain());
lib/CodeGen/SelectionDAG/InstrEmitter.cpp 87 unsigned SrcReg, DenseMap<SDValue, unsigned> &VRBaseMap) {
91 SDValue Op(Node, ResNo);
124 SDValue Op = User->getOperand(i);
181 SDValue Op(Node, ResNo);
193 DenseMap<SDValue, unsigned> &VRBaseMap) {
254 SDValue Op(Node, i);
266 unsigned InstrEmitter::getVR(SDValue Op,
267 DenseMap<SDValue, unsigned> &VRBaseMap) {
281 DenseMap<SDValue, unsigned>::iterator I = VRBaseMap.find(Op);
292 SDValue Op,
295 DenseMap<SDValue, unsigned> &VRBaseMap,
363 SDValue Op,
366 DenseMap<SDValue, unsigned> &VRBaseMap,
476 DenseMap<SDValue, unsigned> &VRBaseMap,
550 SDValue N0 = Node->getOperand(0);
551 SDValue N1 = Node->getOperand(1);
552 SDValue N2 = Node->getOperand(2);
597 SDValue Op(Node, 0);
609 DenseMap<SDValue, unsigned> &VRBaseMap) {
620 SDValue Op(Node, 0);
629 DenseMap<SDValue, unsigned> &VRBaseMap,
646 SDValue Op = Node->getOperand(i);
668 SDValue Op(Node, 0);
678 DenseMap<SDValue, unsigned> &VRBaseMap) {
716 SDValue Op = SDValue(Node, SD->getResNo());
722 DenseMap<SDValue, unsigned>::iterator I = VRBaseMap.find(Op);
783 DenseMap<SDValue, unsigned> &VRBaseMap) {
981 DenseMap<SDValue, unsigned> &VRBaseMap) {
995 SDValue SrcVal = Node->getOperand(2);
1058 SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString);
1146 SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode);
lib/CodeGen/SelectionDAG/InstrEmitter.h 43 DenseMap<SDValue, unsigned> &VRBaseMap);
49 DenseMap<SDValue, unsigned> &VRBaseMap);
53 unsigned getVR(SDValue Op,
54 DenseMap<SDValue, unsigned> &VRBaseMap);
60 SDValue Op,
63 DenseMap<SDValue, unsigned> &VRBaseMap,
71 SDValue Op,
74 DenseMap<SDValue, unsigned> &VRBaseMap,
85 void EmitSubregNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap,
93 DenseMap<SDValue, unsigned> &VRBaseMap);
97 void EmitRegSequence(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap,
108 DenseMap<SDValue, unsigned> &VRBaseMap);
116 DenseMap<SDValue, unsigned> &VRBaseMap) {
135 DenseMap<SDValue, unsigned> &VRBaseMap);
137 DenseMap<SDValue, unsigned> &VRBaseMap);
lib/CodeGen/SelectionDAG/LegalizeDAG.cpp 64 SDValue Chain;
65 SDValue FloatPtr;
66 SDValue IntPtr;
69 SDValue IntValue;
114 SDValue OptimizeFloatStore(StoreSDNode *ST);
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
123 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
125 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
132 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 SDValue N1, SDValue N2,
133 SDValue N1, SDValue N2,
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
136 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
139 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
141 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
141 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
143 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
147 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
153 SDValue ExpandArgFPLibCall(SDNode *Node,
157 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
160 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
163 const SDLoc &dl, SDValue ChainIn);
164 SDValue ExpandBUILD_VECTOR(SDNode *Node);
165 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
166 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
168 SmallVectorImpl<SDValue> &Results);
170 SDValue Value) const;
171 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
172 SDValue NewIntValue) const;
173 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
174 SDValue ExpandFABS(SDNode *Node) const;
175 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
175 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
177 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
177 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
179 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
179 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
182 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
182 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
183 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
183 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
185 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
185 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
186 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
186 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
187 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
189 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
190 SDValue ExpandConstant(ConstantSDNode *CP);
219 void ReplaceNode(SDValue Old, SDValue New) {
219 void ReplaceNode(SDValue Old, SDValue New) {
229 void ReplaceNode(SDNode *Old, const SDValue *New) {
242 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
242 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
259 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
260 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
260 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
288 SDValue
329 SDValue CPIdx =
333 SDValue Result = DAG.getExtLoad(
339 SDValue Result = DAG.getLoad(
346 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
349 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
352 SDValue Result = DAG.getLoad(
362 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
362 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
363 SDValue Val,
364 SDValue Idx,
366 SDValue Tmp1 = Vec;
367 SDValue Tmp2 = Val;
368 SDValue Tmp3 = Idx;
378 SDValue StackPtr = DAG.CreateStackTemporary(VT);
383 SDValue Ch = DAG.getStore(
387 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
396 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
396 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
396 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
397 SDValue Idx,
406 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
423 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
432 SDValue Chain = ST->getChain();
433 SDValue Ptr = ST->getBasePtr();
441 SDValue Con = DAG.getConstant(CFP->getValueAPF().
451 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
462 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
463 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
484 SDValue Chain = ST->getChain();
485 SDValue Ptr = ST->getBasePtr();
499 SDValue Value = ST->getValue();
511 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
519 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
529 SDValue Result =
540 SDValue Value = ST->getValue();
552 SDValue Result =
569 SDValue Lo, Hi;
614 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
625 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
631 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
640 SDValue Result;
665 SDValue Chain = LD->getChain(); // The chain.
666 SDValue Ptr = LD->getBasePtr(); // The base pointer.
667 SDValue Value; // The value returned by the load op.
674 SDValue RVal = SDValue(Node, 0);
675 SDValue RChain = SDValue(Node, 1);
691 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
702 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
743 SDValue Ch;
751 SDValue Result =
783 SDValue Lo, Hi, Ch;
864 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
893 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
911 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
930 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
934 SDValue ValRes;
977 for (const SDValue &Op : Node->op_values())
1100 SDValue NewVal;
1178 SDValue Op0 = Node->getOperand(0);
1179 SDValue Op1 = Node->getOperand(1);
1181 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1199 SDValue Op0 = Node->getOperand(0);
1200 SDValue Op1 = Node->getOperand(1);
1201 SDValue Op2 = Node->getOperand(2);
1203 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1226 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1237 SmallVector<SDValue, 8> ResultVals;
1278 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1278 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1279 SDValue Vec = Op.getOperand(0);
1280 SDValue Idx = Op.getOperand(1);
1295 SDValue StackPtr, Ch;
1335 SDValue NewLoad;
1350 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1358 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1358 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1361 SDValue Vec = Op.getOperand(0);
1362 SDValue Part = Op.getOperand(1);
1363 SDValue Idx = Op.getOperand(2);
1368 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1374 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1377 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1386 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1394 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1400 SmallVector<SDValue, 8> Stores;
1410 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1424 SDValue StoreChain;
1439 SDValue Value) const {
1456 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1465 SDValue IntPtr;
1489 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1491 SDValue NewIntValue) const {
1496 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1502 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1504 SDValue Mag = Node->getOperand(0);
1505 SDValue Sign = Node->getOperand(1);
1512 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1513 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1520 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1521 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1522 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1531 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1532 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1543 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1546 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1554 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1558 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1560 SDValue Value = Node->getOperand(0);
1565 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1573 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1574 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1580 SmallVectorImpl<SDValue> &Results) {
1586 SDValue Tmp1 = SDValue(Node, 0);
1587 SDValue Tmp2 = SDValue(Node, 1);
1588 SDValue Tmp3 = Node->getOperand(2);
1589 SDValue Chain = Tmp1.getOperand(0);
1595 SDValue Size = Tmp2.getOperand(1);
1596 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1632 bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1633 SDValue &RHS, SDValue &CC,
1633 SDValue &RHS, SDValue &CC,
1714 SDValue SetCC1, SetCC2;
1738 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1738 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1743 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1743 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1745 SDValue Chain) {
1749 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1764 SDValue Store;
1784 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1788 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1793 SDValue Ch = DAG.getTruncStore(
1804 const TargetLowering &TLI, SDValue &Res) {
1817 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1820 SDValue V = Node->getOperand(i);
1824 SDValue Vec;
1853 SDValue Shuffle;
1874 SDValue Vec1 = IntermedVals[0].first;
1875 SDValue Vec2;
1898 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1900 SDValue Value1, Value2;
1912 SDValue V = Node->getOperand(i);
1962 SDValue CPIdx =
1971 SmallSet<SDValue, 16> DefinedValues;
1982 SDValue V = Node->getOperand(i);
1989 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1990 SDValue Vec2;
2000 SDValue Res;
2010 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2013 SDValue SplatVal = Node->getOperand(0);
2022 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2026 for (const SDValue &Op : Node->op_values()) {
2035 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2045 SDValue InChain = DAG.getEntryNode();
2049 SDValue TCChain = InChain;
2068 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2068 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2082 std::pair<SDValue, SDValue>
2082 std::pair<SDValue, SDValue>
2086 SDValue InChain = Node->getOperand(0);
2099 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2112 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2112 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2117 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2138 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2158 SDValue SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2183 SmallVectorImpl<SDValue> &Results) {
2200 SDValue InChain = DAG.getEntryNode();
2207 for (const SDValue &Op : Node->op_values()) {
2218 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2225 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2237 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2237 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2240 SDValue Rem =
2265 SDValue Op0 = Node->getOperand(0);
2281 SmallVectorImpl<SDValue> &Results) {
2295 SDValue InChain = DAG.getEntryNode();
2311 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2319 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2326 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2335 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2335 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2347 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2347 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2359 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2362 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2365 SDValue Hi = StackSlot;
2366 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2372 SDValue Op0Mapped;
2375 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2381 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2384 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2386 SDValue Store2 =
2389 SDValue Load =
2392 SDValue Bias = DAG.getConstantFP(isSigned ?
2397 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2399 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2405 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2407 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2409 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2411 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2430 SDValue CPIdx =
2435 SDValue FudgeInReg;
2442 SDValue Load = DAG.getExtLoad(
2459 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2459 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2500 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2500 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2530 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2538 SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2538 SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2543 SDValue Tmp, Tmp2, Tmp3;
2602 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2602 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2605 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2655 SmallVector<SDValue, 8> Results;
2657 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2690 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2692 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2697 SDValue FA = DAG.getNode(
2731 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2733 SDValue Swap = DAG.getAtomicCmpSwap(
2743 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2756 SDValue Res = DAG.getAtomicCmpSwap(
2761 SDValue ExtRes = Res;
2762 SDValue LHS = Res;
2763 SDValue RHS = Node->getOperand(1);
2789 SDValue Success =
2865 SDValue One = DAG.getConstant(1, dl, VT);
2866 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2867 SDValue Zero = DAG.getConstant(0, dl, VT);
2868 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2878 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2960 SDValue Op0 = Node->getOperand(0);
2961 SDValue Op1 = Node->getOperand(1);
3006 SmallVector<SDValue, 16> Ops;
3109 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3137 SDValue Res =
3146 SDValue Op = Node->getOperand(0);
3152 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3244 SDValue LHS = Node->getOperand(0);
3245 SDValue RHS = Node->getOperand(1);
3256 SmallVector<SDValue, 4> Halves;
3263 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3264 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3265 SDValue Shift = DAG.getConstant(
3303 SDValue Lo, Hi;
3313 SDValue Shift =
3345 SDValue LHS = Node->getOperand(0);
3346 SDValue RHS = Node->getOperand(1);
3347 SDValue Carry = Node->getOperand(2);
3354 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3360 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3363 SDValue CarryExt =
3365 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3371 SDValue Zero = DAG.getConstant(0, dl, VT);
3372 SDValue Overflow2 =
3378 SDValue ResultCarry =
3387 SDValue Result, Overflow;
3395 SDValue Result, Overflow;
3403 SDValue Result, Overflow;
3438 SDValue Chain = Node->getOperand(0);
3439 SDValue Table = Node->getOperand(1);
3440 SDValue Index = Node->getOperand(2);
3459 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3463 SDValue LD = DAG.getExtLoad(
3556 SDValue CC = Node->getOperand(4);
3567 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3667 SmallVector<SDValue, 8> Scalars;
3669 SDValue Ex = DAG.getNode(
3672 SDValue Sh = DAG.getNode(
3679 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3751 SmallVector<SDValue, 8> Results;
3771 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3771 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3796 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3796 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3811 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3811 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4123 SmallVector<SDValue, 8> Results;
4136 SDValue Tmp1, Tmp2, Tmp3;
4192 SDValue Chain = Node->getOperand(0); // Get the chain.
4193 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4423 SmallVector<SDValue, 8> NewOps;
4425 SDValue Op = Node->getOperand(I);
4430 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4431 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4457 SDValue Idx = Node->getOperand(1);
4460 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4461 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4463 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4465 SmallVector<SDValue, 8> NewOps;
4467 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4468 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4470 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4475 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4503 SDValue Val = Node->getOperand(1);
4504 SDValue Idx = Node->getOperand(2);
4508 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4509 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4511 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4512 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4514 SDValue NewVec = CastVec;
4516 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4517 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4519 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4541 SDValue Val = Node->getOperand(0);
4544 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4545 SDValue Undef = DAG.getUNDEF(MidVT);
4547 SmallVector<SDValue, 8> NewElts;
4552 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4553 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4560 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4566 SDValue NewAtomic
lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp 51 SDValue R = SDValue();
115 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
119 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
121 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
125 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
134 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
158 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
159 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
165 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
172 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
173 SDValue Op = GetSoftenedFloat(N->getOperand(0));
177 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
179 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
194 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
196 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
211 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
213 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
228 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
230 SDValue Op = GetSoftenedFloat(N->getOperand(0));
243 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
245 SDValue Op = GetSoftenedFloat(N->getOperand(0));
258 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
259 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
260 SDValue RHS = BitConvertToInteger(N->getOperand(1));
270 SDValue SignBit = DAG.getNode(
295 SDValue Mask = DAG.getNode(
306 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
308 SDValue Op = GetSoftenedFloat(N->getOperand(0));
321 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
323 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
338 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
340 SDValue Op = GetSoftenedFloat(N->getOperand(0));
353 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
355 SDValue Op = GetSoftenedFloat(N->getOperand(0));
368 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
370 SDValue Op = GetSoftenedFloat(N->getOperand(0));
383 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
385 SDValue Op = GetSoftenedFloat(N->getOperand(0));
398 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
400 SDValue Op = GetSoftenedFloat(N->getOperand(0));
413 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
415 SDValue Op = GetSoftenedFloat(N->getOperand(0));
428 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
430 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0)),
447 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
449 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
464 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
466 SDValue Op = GetSoftenedFloat(N->getOperand(0));
479 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
492 SDValue Ops[2] = { DAG.getConstantFP(-0.0, dl, N->getValueType(0)),
504 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
506 SDValue Op = N->getOperand(0);
536 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
538 SDValue Op = N->getOperand(0);
542 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
553 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
555 SDValue Op = N->getOperand(0);
570 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
572 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
587 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
591 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), N->getOperand(1) };
605 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
607 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
622 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
624 SDValue Op = GetSoftenedFloat(N->getOperand(0));
637 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
639 SDValue Op = GetSoftenedFloat(N->getOperand(0));
652 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
654 SDValue Op = GetSoftenedFloat(N->getOperand(0));
667 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
669 SDValue Op = GetSoftenedFloat(N->getOperand(0));
682 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
684 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)),
699 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
704 SDValue Op = GetSoftenedFloat(N->getOperand(0));
717 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
726 SDValue NewL;
747 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
751 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
752 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
753 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
758 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
759 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
760 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
766 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
771 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
772 SDValue Chain = N->getOperand(0); // Get the chain.
773 SDValue Ptr = N->getOperand(1); // Get the pointer.
778 SDValue NewVAARG;
789 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
810 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
829 SDValue Res = SDValue();
870 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
871 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
876 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_EXTEND(SDNode *N) {
880 SDValue Op = GetSoftenedFloat(N->getOperand(0));
895 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
907 SDValue Op = GetSoftenedFloat(N->getOperand(0));
914 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
915 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
938 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
960 SDValue Op = GetSoftenedFloat(N->getOperand(0));
964 SDValue Res = TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl).first;
970 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
971 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
994 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
995 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1017 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1021 SDValue Val = ST->getValue();
1035 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1038 SDValue Op = GetSoftenedFloat(N->getOperand(0));
1052 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1055 SDValue Op = GetSoftenedFloat(N->getOperand(0));
1069 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1072 SDValue Op = GetSoftenedFloat(N->getOperand(0));
1086 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1089 SDValue Op = GetSoftenedFloat(N->getOperand(0));
1113 SDValue Lo, Hi;
1179 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1180 SDValue &Hi) {
1194 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1195 SDValue &Hi) {
1199 SDValue Tmp;
1208 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1209 SDValue &Hi) {
1210 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1218 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1219 SDValue &Hi) {
1220 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1228 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1229 SDValue &Hi) {
1230 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1238 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1239 SDValue &Hi) {
1240 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1248 SDValue &Lo, SDValue &Hi) {
1248 SDValue &Lo, SDValue &Hi) {
1249 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1258 SDValue &Lo, SDValue &Hi) {
1258 SDValue &Lo, SDValue &Hi) {
1259 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1270 SDValue &Lo, SDValue &Hi) {
1270 SDValue &Lo, SDValue &Hi) {
1271 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1279 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1280 SDValue &Hi) {
1281 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
1283 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1295 SDValue &Lo, SDValue &Hi) {
1295 SDValue &Lo, SDValue &Hi) {
1296 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1305 SDValue &Lo, SDValue &Hi) {
1305 SDValue &Lo, SDValue &Hi) {
1306 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1315 SDValue &Lo, SDValue &Hi) {
1315 SDValue &Lo, SDValue &Hi) {
1316 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1325 SDValue &Lo, SDValue &Hi) {
1325 SDValue &Lo, SDValue &Hi) {
1326 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1335 SDValue &Lo, SDValue &Hi) {
1335 SDValue &Lo, SDValue &Hi) {
1336 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1345 SDValue &Lo, SDValue &Hi) {
1345 SDValue &Lo, SDValue &Hi) {
1346 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1354 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1355 SDValue &Hi) {
1356 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) };
1358 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1369 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1370 SDValue &Hi) {
1371 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
1373 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1385 SDValue &Lo, SDValue &Hi) {
1385 SDValue &Lo, SDValue &Hi) {
1386 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1396 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1397 SDValue &Hi) {
1404 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1405 SDValue &Hi) {
1414 SDValue &Lo, SDValue &Hi) {
1414 SDValue &Lo, SDValue &Hi) {
1415 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1424 SDValue &Lo, SDValue &Hi) {
1424 SDValue &Lo, SDValue &Hi) {
1425 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1434 SDValue &Lo, SDValue &Hi) {
1434 SDValue &Lo, SDValue &Hi) {
1435 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1444 SDValue &Lo, SDValue &Hi) {
1444 SDValue &Lo, SDValue &Hi) {
1445 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1454 SDValue &Lo, SDValue &Hi) {
1454 SDValue &Lo, SDValue &Hi) {
1455 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1466 SDValue &Lo, SDValue &Hi) {
1466 SDValue &Lo, SDValue &Hi) {
1467 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1476 SDValue &Lo, SDValue &Hi) {
1476 SDValue &Lo, SDValue &Hi) {
1477 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1485 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1486 SDValue &Hi) {
1487 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
1489 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1501 SDValue &Lo, SDValue &Hi) {
1501 SDValue &Lo, SDValue &Hi) {
1502 SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0),
1510 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1511 SDValue &Hi) {
1519 SDValue Chain = LD->getChain();
1520 SDValue Ptr = LD->getBasePtr();
1542 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1543 SDValue &Hi) {
1547 SDValue Src = N->getOperand(0);
1628 SDValue Res = SDValue();
1678 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1679 SDValue &NewRHS,
1682 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1693 SDValue Tmp1, Tmp2, Tmp3;
1708 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1709 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1726 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1729 SDValue Lo, Hi;
1737 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1740 SDValue Lo, Hi;
1747 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_SINT(SDNode *N) {
1757 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) {
1768 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1769 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1786 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1787 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1803 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
1811 SDValue Chain = ST->getChain();
1812 SDValue Ptr = ST->getBasePtr();
1820 SDValue Lo, Hi;
1827 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
1840 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
1853 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
1866 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
1897 SDValue R = SDValue();
1932 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
1933 SDValue Op = N->getOperand(0);
1936 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
1941 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
1950 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
1952 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
1959 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
1960 SDValue Op = GetPromotedFloat(N->getOperand(0));
1964 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
1965 SDValue Op = GetPromotedFloat(N->getOperand(0));
1979 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1980 SDValue LHS = GetPromotedFloat(N->getOperand(0));
1981 SDValue RHS = GetPromotedFloat(N->getOperand(1));
1990 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
1993 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
1994 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2003 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2005 SDValue Val = ST->getValue();
2008 SDValue Promoted = GetPromotedFloat(Val);
2012 SDValue NewVal;
2027 SDValue R = SDValue();
2110 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2117 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2121 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2128 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2142 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2148 SDValue Vec = N->getOperand(0);
2149 SDValue Idx = N->getOperand(1);
2158 SDValue Res = GetScalarizedVector(N->getOperand(0));
2164 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2169 SDValue Lo, Hi;
2173 SDValue Res;
2188 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2192 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2204 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2207 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2209 SDValue Op1 = N->getOperand(1);
2217 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2220 SDValue Op = GetPromotedFloat(N->getOperand(0));
2228 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2231 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2232 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2236 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2239 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2240 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2241 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2247 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2250 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2251 SDValue Op1 = N->getOperand(1);
2258 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2261 SDValue Op = N->getOperand(0);
2268 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2273 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2279 SDValue newL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), IVT,
2295 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2296 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2297 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2305 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2306 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2307 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2316 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2320 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2327 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2332 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2339 SDValue CastVal = BitConvertToInteger(AM->getVal());
2342 SDValue NewAtomic
2348 SDValue ResultCast = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp 40 SDValue Res = SDValue();
203 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
205 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
209 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
211 SDValue Op = SExtPromotedInteger(N->getOperand(0));
216 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
218 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
223 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
225 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
235 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
236 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
237 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
247 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
260 SDValue Res = DAG.getAtomicCmpSwap(
269 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
270 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
273 SDValue Res = DAG.getAtomicCmpSwap(
282 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
283 SDValue InOp = N->getOperand(0);
320 SDValue Lo, Hi;
381 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
382 SDValue Op = GetPromotedInteger(N->getOperand(0));
393 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
394 SDValue Op = GetPromotedInteger(N->getOperand(0));
406 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
415 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
422 SDValue Result = DAG.getNode(Opc, dl,
429 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
431 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
443 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
445 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
449 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
450 SDValue Op = GetPromotedInteger(N->getOperand(0));
465 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
469 SDValue Op0 = N->getOperand(0);
470 SDValue Op1 = N->getOperand(1);
476 SDValue In = GetPromotedInteger(Op0);
482 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
490 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
509 SDValue Res;
532 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
539 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
546 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
552 SDValue Res = GetPromotedInteger(N->getOperand(0));
574 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
580 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
589 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
591 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
594 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
603 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
605 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
610 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
612 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
622 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
628 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
635 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
646 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
655 SDValue Op1 = N->getOperand(0);
656 SDValue Op2 = N->getOperand(1);
661 SDValue Op1Promoted, Op2Promoted;
690 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
696 SDValue Result =
701 SDValue Max =
708 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
709 SDValue Add =
717 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
718 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
719 SDValue Result =
727 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
730 SDValue Op1Promoted, Op2Promoted;
755 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
765 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
771 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
772 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
779 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
783 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
794 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
795 SDValue LHS = GetPromotedInteger(N->getOperand(1));
796 SDValue RHS = GetPromotedInteger(N->getOperand(2));
801 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
802 SDValue Mask = N->getOperand(0);
804 SDValue LHS = GetPromotedInteger(N->getOperand(1));
805 SDValue RHS = GetPromotedInteger(N->getOperand(2));
810 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
811 SDValue LHS = GetPromotedInteger(N->getOperand(2));
812 SDValue RHS = GetPromotedInteger(N->getOperand(3));
818 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
842 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
849 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
850 SDValue LHS = GetPromotedInteger(N->getOperand(0));
851 SDValue RHS = N->getOperand(1);
857 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
858 SDValue Op = GetPromotedInteger(N->getOperand(0));
863 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
867 SDValue LHS = GetPromotedInteger(N->getOperand(0));
868 SDValue RHS = GetPromotedInteger(N->getOperand(1));
873 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
875 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
876 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
881 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
883 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
884 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
889 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
891 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
892 SDValue RHS = N->getOperand(1);
898 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
900 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
901 SDValue RHS = N->getOperand(1);
907 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
909 SDValue Res;
910 SDValue InOp = N->getOperand(0);
931 SDValue EOp1, EOp2;
942 SDValue WideInOp = GetWidenedVector(InOp);
948 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
953 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
957 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
966 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
972 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
973 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
980 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
984 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
997 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1011 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1012 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1017 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1026 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1027 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1031 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1036 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1052 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1057 SDValue Overflow;
1062 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1069 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1084 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1089 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1090 SDValue Chain = N->getOperand(0); // Get the chain.
1091 SDValue Ptr = N->getOperand(1); // Get the pointer.
1099 SmallVector<SDValue, 8> Parts(NumRegs);
1112 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1114 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1140 SDValue Res = SDValue();
1243 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1243 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1253 SDValue OpL = GetPromotedInteger(NewLHS);
1254 SDValue OpR = GetPromotedInteger(NewRHS);
1291 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1292 SDValue Op = GetPromotedInteger(N->getOperand(0));
1296 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1297 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1302 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1308 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1311 SDValue LHS = N->getOperand(2);
1312 SDValue RHS = N->getOperand(3);
1322 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1326 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1333 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1336 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1337 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1347 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1363 SmallVector<SDValue, 16> NewOps;
1370 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1389 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1395 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1402 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1409 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1411 SDValue Cond = N->getOperand(0);
1415 if (SDValue Res = WidenVSELECTAndMask(N))
1426 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1429 SDValue LHS = N->getOperand(0);
1430 SDValue RHS = N->getOperand(1);
1438 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1441 SDValue LHS = N->getOperand(0);
1442 SDValue RHS = N->getOperand(1);
1449 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1454 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1455 SDValue Op = GetPromotedInteger(N->getOperand(0));
1462 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1467 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1469 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1472 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1479 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1482 SDValue DataOp = N->getValue();
1484 SDValue Mask = N->getMask();
1491 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1505 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1509 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1510 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1515 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1518 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1536 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1538 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1555 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1556 SDValue Op = GetPromotedInteger(N->getOperand(0));
1560 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1565 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1567 SDValue Op = GetPromotedInteger(N->getOperand(0));
1573 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1576 SDValue LHS = N->getOperand(0);
1577 SDValue RHS = N->getOperand(1);
1578 SDValue Carry = N->getOperand(2);
1586 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1587 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1592 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1594 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1598 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1602 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1603 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1604 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1610 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1611 SDValue Op = SExtPromotedInteger(N->getOperand(1));
1615 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1617 SDValue Op;
1644 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1659 SDValue Lo, Hi;
1729 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1729 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1737 SDValue Tmp = DAG.getAtomicCmpSwap(
1745 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1813 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1813 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1825 SDValue &Lo, SDValue &Hi) {
1825 SDValue &Lo, SDValue &Hi) {
1828 SDValue InL, InH;
1914 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1914 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1915 SDValue Amt = N->getOperand(1);
1932 SDValue InL, InH;
1966 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1983 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1985 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2001 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2001 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2002 SDValue Amt = N->getOperand(1);
2011 SDValue InL, InH;
2014 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2015 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2016 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2017 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2019 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2023 SDValue LoS, HiS, LoL, HiL;
2093 SDValue &Lo, SDValue &Hi) {
2093 SDValue &Lo, SDValue &Hi) {
2100 SDValue LHSL, LHSH, RHSL, RHSH;
2113 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2114 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2117 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2120 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2126 SDValue &Lo, SDValue &Hi) {
2126 SDValue &Lo, SDValue &Hi) {
2129 SDValue LHSL, LHSH, RHSL, RHSH;
2134 SDValue LoOps[2] = { LHSL, RHSL };
2135 SDValue HiOps[3] = { LHSH, RHSH };
2197 SDValue OVF = Lo.getValue(1);
2217 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2221 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2226 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2229 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2231 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2237 SDValue Cmp =
2241 SDValue Borrow;
2253 SDValue &Lo, SDValue &Hi) {
2253 SDValue &Lo, SDValue &Hi) {
2255 SDValue LHSL, LHSH, RHSL, RHSH;
2260 SDValue LoOps[2] = { LHSL, RHSL };
2261 SDValue HiOps[3] = { LHSH, RHSH };
2279 SDValue &Lo, SDValue &Hi) {
2279 SDValue &Lo, SDValue &Hi) {
2281 SDValue LHSL, LHSH, RHSL, RHSH;
2286 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2287 SDValue HiOps[3] = { LHSH, RHSH };
2299 SDValue &Lo, SDValue &Hi) {
2299 SDValue &Lo, SDValue &Hi) {
2300 SDValue LHS = N->getOperand(0);
2301 SDValue RHS = N->getOperand(1);
2304 SDValue Ovf;
2328 SDValue LHSL, LHSH, RHSL, RHSH;
2332 SDValue LoOps[2] = { LHSL, RHSL };
2333 SDValue HiOps[3] = { LHSH, RHSH };
2343 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2357 SDValue &Lo, SDValue &Hi) {
2357 SDValue &Lo, SDValue &Hi) {
2359 SDValue LHSL, LHSH, RHSL, RHSH;
2364 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2365 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2377 SDValue &Lo, SDValue &Hi) {
2377 SDValue &Lo, SDValue &Hi) {
2380 SDValue Op = N->getOperand(0);
2391 SDValue Res = GetPromotedInteger(Op);
2400 SDValue &Lo, SDValue &Hi) {
2400 SDValue &Lo, SDValue &Hi) {
2422 SDValue &Lo, SDValue &Hi) {
2422 SDValue &Lo, SDValue &Hi) {
2442 SDValue &Lo, SDValue &Hi) {
2442 SDValue &Lo, SDValue &Hi) {
2450 SDValue &Lo, SDValue &Hi) {
2450 SDValue &Lo, SDValue &Hi) {
2458 SDValue &Lo, SDValue &Hi) {
2458 SDValue &Lo, SDValue &Hi) {
2471 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2471 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2476 SDValue N0 = N->getOperand(0);
2477 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2479 SDValue NegLo, NegHi;
2484 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2491 SDValue &Lo, SDValue &Hi) {
2491 SDValue &Lo, SDValue &Hi) {
2497 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2500 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2501 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2511 SDValue &Lo, SDValue &Hi) {
2511 SDValue &Lo, SDValue &Hi) {
2522 SDValue &Lo, SDValue &Hi) {
2522 SDValue &Lo, SDValue &Hi) {
2528 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2531 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2532 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2541 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2542 SDValue &Hi) {
2554 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2555 SDValue &Hi) {
2559 SDValue Op = N->getOperand(0);
2571 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2572 SDValue &Hi) {
2576 SDValue Op = N->getOperand(0);
2587 void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo,
2588 SDValue &Hi) {
2603 SDValue Op = N->getOperand(0);
2615 void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo,
2616 SDValue &Hi) {
2631 SDValue Op = N->getOperand(0);
2644 SDValue &Lo, SDValue &Hi) {
2644 SDValue &Lo, SDValue &Hi) {
2654 SDValue Ch = N->getChain();
2655 SDValue Ptr = N->getBasePtr();
2758 SDValue &Lo, SDValue &Hi) {
2758 SDValue &Lo, SDValue &Hi) {
2760 SDValue LL, LH, RL, RH;
2768 SDValue &Lo, SDValue &Hi) {
2768 SDValue &Lo, SDValue &Hi) {
2773 SDValue LL, LH, RL, RH;
2800 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2802 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2803 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2805 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2806 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2814 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2815 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2816 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2817 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2819 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2821 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2822 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2824 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2826 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2828 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2841 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2848 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2849 SDValue &Hi) {
2853 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2859 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2860 SDValue &Hi) {
2861 SDValue Result = TLI.expandAddSubSat(N, DAG);
2870 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2871 SDValue &Hi) {
2875 SDValue LHS = N->getOperand(0);
2876 SDValue RHS = N->getOperand(1);
2885 SDValue Result;
2892 SDValue Product = Result.getValue(0);
2893 SDValue Overflow = Result.getValue(1);
2897 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2898 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2899 SDValue Zero = DAG.getConstant(0, dl, VT);
2900 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2907 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2920 SDValue LL, LH, RL, RH;
2923 SmallVector<SDValue, 4> Result;
2959 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
2989 SDValue ResultHL = Result[2];
2990 SDValue ResultHH = Result[3];
2992 SDValue SatMax, SatMin;
2993 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2994 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3000 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3002 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3009 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3030 SDValue HLHiMask = DAG.getConstant(
3032 SDValue HLLoMask = DAG.getConstant(
3035 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3036 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3037 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3041 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3042 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3043 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3048 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3049 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3050 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3054 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3055 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3056 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3063 SDValue HHHiMask = DAG.getConstant(
3065 SDValue HHLoMask = DAG.getConstant(
3084 SDValue &Lo, SDValue &Hi) {
3084 SDValue &Lo, SDValue &Hi) {
3085 SDValue LHS = Node->getOperand(0);
3086 SDValue RHS = Node->getOperand(1);
3091 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3108 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3110 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3111 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3112 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3116 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3117 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3119 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3126 SDValue &Lo, SDValue &Hi) {
3126 SDValue &Lo, SDValue &Hi) {
3129 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3132 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3154 SDValue &Lo, SDValue &Hi) {
3154 SDValue &Lo, SDValue &Hi) {
3190 SDValue LHSL, LHSH;
3197 SDValue ShiftOp = N->getOperand(1);
3205 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3248 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3260 SDValue &Lo, SDValue &Hi) {
3260 SDValue &Lo, SDValue &Hi) {
3263 SDValue Op = N->getOperand(0);
3278 SDValue Res = GetPromotedInteger(Op);
3291 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3291 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3317 SDValue &Lo, SDValue &Hi) {
3317 SDValue &Lo, SDValue &Hi) {
3320 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3323 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3345 SDValue &Lo, SDValue &Hi) {
3345 SDValue &Lo, SDValue &Hi) {
3357 SDValue &Lo, SDValue &Hi) {
3357 SDValue &Lo, SDValue &Hi) {
3374 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3375 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3383 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3384 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3388 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3390 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3393 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3395 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3404 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3407 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3408 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3429 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3431 SDValue Chain =
3437 for (const SDValue &Op : N->op_values()) {
3454 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3462 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3462 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3465 SDValue Temp2 =
3467 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3475 SDValue &Lo, SDValue &Hi) {
3475 SDValue &Lo, SDValue &Hi) {
3478 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3481 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3502 SDValue &Lo, SDValue &Hi) {
3502 SDValue &Lo, SDValue &Hi) {
3505 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3508 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3529 SDValue &Lo, SDValue &Hi) {
3529 SDValue &Lo, SDValue &Hi) {
3532 SDValue Op = N->getOperand(0);
3543 SDValue Res = GetPromotedInteger(Op);
3556 SDValue &Lo, SDValue &Hi) {
3556 SDValue &Lo, SDValue &Hi) {
3560 SDValue Zero = DAG.getConstant(0, dl, VT);
3561 SDValue Swap = DAG.getAtomicCmpSwap(
3571 SDValue &Lo, SDValue &Hi) {
3571 SDValue &Lo, SDValue &Hi) {
3574 SDValue Res = TLI.expandVecReduce(N, DAG);
3589 SDValue Res = SDValue();
3644 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3645 SDValue &NewRHS,
3648 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3704 SDValue LoCmp, HiCmp;
3772 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3773 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3790 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3791 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3808 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3809 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3826 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3827 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3843 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3844 SDValue LHS = N->getOperand(0);
3845 SDValue RHS = N->getOperand(1);
3846 SDValue Carry = N->getOperand(2);
3847 SDValue Cond = N->getOperand(3);
3850 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3856 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3861 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3865 SDValue Lo, Hi;
3870 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3874 SDValue Lo, Hi;
3879 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3880 SDValue Op = N->getOperand(0);
3890 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3899 SDValue Ch = N->getChain();
3900 SDValue Ptr = N->getBasePtr();
3905 SDValue Lo, Hi;
3972 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3973 SDValue InL, InH;
3979 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3980 SDValue Op = N->getOperand(0);
3992 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
4014 SDValue Lo, Hi;
4016 SDValue SignSet = DAG.getSetCC(dl,
4023 SDValue FudgePtr =
4028 SDValue Zero = DAG.getIntPtrConstant(0, dl);
4029 SDValue Four = DAG.getIntPtrConstant(4, dl);
4032 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
4041 SDValue Fudge = DAG.getExtLoad(
4057 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4059 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4068 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4077 SDValue BaseIdx = N->getOperand(1);
4079 SDValue InOp0 = N->getOperand(0);
4085 SmallVector<SDValue, 8> Ops;
4090 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4092 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4095 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4104 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4111 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4112 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4119 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4128 SmallVector<SDValue, 8> Ops;
4131 SDValue Op;
4146 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4158 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4163 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
4166 SDValue SplatVal = N->getOperand(0);
4175 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4180 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4196 SmallVector<SDValue, 8> Ops(NumOutElem);
4198 SDValue Op = N->getOperand(i);
4206 SDValue Ext = DAG.getNode(
4216 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4229 SDValue Promoted;
4251 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4259 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4261 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4267 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4275 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4277 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4278 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4280 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4289 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4291 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4295 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4299 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4305 SmallVector<SDValue, 8> NewOps;
4310 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4316 SDValue Ex = DAG.getNode(
4319 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
lib/CodeGen/SelectionDAG/LegalizeTypes.cpp 83 SDValue Res(&Node, i);
106 SDValue NewVal = getSDValue(NewValId);
292 const auto Op = N->getOperand(i);
492 std::vector<SDValue> NewOps;
495 SDValue OrigOp = N->getOperand(i);
496 SDValue Op = OrigOp;
544 void DAGTypeLegalizer::AnalyzeNewValue(SDValue &Val) {
553 void DAGTypeLegalizer::RemapValue(SDValue &V) {
622 void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) {
622 void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) {
662 SDValue OldVal(N, i);
663 SDValue NewVal(M, i);
685 void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) {
685 void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) {
699 void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
699 void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
710 void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
710 void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
721 void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
721 void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
734 void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo,
734 void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo,
735 SDValue &Hi) {
742 void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo,
742 void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo,
743 SDValue Hi) {
771 void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo,
771 void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo,
772 SDValue &Hi) {
779 void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo,
779 void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo,
780 SDValue Hi) {
795 void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo,
795 void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo,
796 SDValue &Hi) {
804 void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
804 void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
805 SDValue Hi) {
823 void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
823 void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
840 SDValue DAGTypeLegalizer::BitConvertToInteger(SDValue Op) {
840 SDValue DAGTypeLegalizer::BitConvertToInteger(SDValue Op) {
847 SDValue DAGTypeLegalizer::BitConvertVectorToIntegerVector(SDValue Op) {
847 SDValue DAGTypeLegalizer::BitConvertVectorToIntegerVector(SDValue Op) {
856 SDValue DAGTypeLegalizer::CreateStackStoreLoad(SDValue Op,
856 SDValue DAGTypeLegalizer::CreateStackStoreLoad(SDValue Op,
861 SDValue StackPtr = DAG.CreateStackTemporary(Op.getValueType(), DestVT);
863 SDValue Store =
882 SmallVector<SDValue, 8> Results;
920 SmallVector<SDValue, 8> Results;
940 SDValue DAGTypeLegalizer::DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo) {
949 void DAGTypeLegalizer::GetPairElements(SDValue Pair,
950 SDValue &Lo, SDValue &Hi) {
950 SDValue &Lo, SDValue &Hi) {
960 SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
960 SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
960 SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
978 SDValue DAGTypeLegalizer::LibCallify(RTLIB::Libcall LC, SDNode *N,
988 SDValue Op = N->getOperand(0);
992 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
996 SmallVector<SDValue, 8> Ops(NumOps);
1005 std::pair<SDValue, SDValue>
1005 std::pair<SDValue, SDValue>
1008 SDValue InChain = Node->getOperand(0);
1021 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1034 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
1034 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
1044 SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
1044 SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
1053 void DAGTypeLegalizer::SplitInteger(SDValue Op,
1055 SDValue &Lo, SDValue &Hi) {
1055 SDValue &Lo, SDValue &Hi) {
1073 void DAGTypeLegalizer::SplitInteger(SDValue Op,
1074 SDValue &Lo, SDValue &Hi) {
1074 SDValue &Lo, SDValue &Hi) {
lib/CodeGen/SelectionDAG/LegalizeTypes.h 93 SmallDenseMap<SDValue, TableId, 8> ValueToIdMap;
94 SmallDenseMap<TableId, SDValue, 8> IdToValueMap;
136 TableId getTableId(SDValue V) {
155 const SDValue &getSDValue(TableId &Id) {
200 void AnalyzeNewValue(SDValue &Val);
203 void RemapValue(SDValue &V);
206 SDValue BitConvertToInteger(SDValue Op);
206 SDValue BitConvertToInteger(SDValue Op);
207 SDValue BitConvertVectorToIntegerVector(SDValue Op);
207 SDValue BitConvertVectorToIntegerVector(SDValue Op);
208 SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
208 SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
215 SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
217 SDValue JoinIntegers(SDValue Lo, SDValue Hi);
217 SDValue JoinIntegers(SDValue Lo, SDValue Hi);
217 SDValue JoinIntegers(SDValue Lo, SDValue Hi);
218 SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
220 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
220 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
222 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
222 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
224 SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
224 SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
226 void ReplaceValueWith(SDValue From, SDValue To);
226 void ReplaceValueWith(SDValue From, SDValue To);
227 void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
227 void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
227 void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
228 void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT,
229 SDValue &Lo, SDValue &Hi);
229 SDValue &Lo, SDValue &Hi);
249 SDValue GetPromotedInteger(SDValue Op) {
249 SDValue GetPromotedInteger(SDValue Op) {
251 SDValue PromotedOp = getSDValue(PromotedId);
255 void SetPromotedInteger(SDValue Op, SDValue Result);
255 void SetPromotedInteger(SDValue Op, SDValue Result);
258 SDValue SExtPromotedInteger(SDValue Op) {
258 SDValue SExtPromotedInteger(SDValue Op) {
267 SDValue ZExtPromotedInteger(SDValue Op) {
267 SDValue ZExtPromotedInteger(SDValue Op) {
278 SDValue SExtOrZExtPromotedInteger(SDValue Op) {
278 SDValue SExtOrZExtPromotedInteger(SDValue Op) {
290 SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
291 SDValue PromoteIntRes_AssertSext(SDNode *N);
292 SDValue PromoteIntRes_AssertZext(SDNode *N);
293 SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
294 SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
295 SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
296 SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
297 SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
298 SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
299 SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
300 SDValue PromoteIntRes_SPLAT_VECTOR(SDNode *N);
301 SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
302 SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
303 SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
304 SDValue PromoteIntRes_BITCAST(SDNode *N);
305 SDValue PromoteIntRes_BSWAP(SDNode *N);
306 SDValue PromoteIntRes_BITREVERSE(SDNode *N);
307 SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
308 SDValue PromoteIntRes_Constant(SDNode *N);
309 SDValue PromoteIntRes_CTLZ(SDNode *N);
310 SDValue PromoteIntRes_CTPOP(SDNode *N);
311 SDValue PromoteIntRes_CTTZ(SDNode *N);
312 SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
313 SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
314 SDValue PromoteIntRes_FP_TO_FP16(SDNode *N);
315 SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
316 SDValue PromoteIntRes_LOAD(LoadSDNode *N);
317 SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
318 SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
319 SDValue PromoteIntRes_Overflow(SDNode *N);
320 SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
321 SDValue PromoteIntRes_SELECT(SDNode *N);
322 SDValue PromoteIntRes_VSELECT(SDNode *N);
323 SDValue PromoteIntRes_SELECT_CC(SDNode *N);
324 SDValue PromoteIntRes_SETCC(SDNode *N);
325 SDValue PromoteIntRes_SHL(SDNode *N);
326 SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
327 SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
328 SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
329 SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
330 SDValue PromoteIntRes_SRA(SDNode *N);
331 SDValue PromoteIntRes_SRL(SDNode *N);
332 SDValue PromoteIntRes_TRUNCATE(SDNode *N);
333 SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
334 SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
335 SDValue PromoteIntRes_UNDEF(SDNode *N);
336 SDValue PromoteIntRes_VAARG(SDNode *N);
337 SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
338 SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
339 SDValue PromoteIntRes_MULFIX(SDNode *N);
340 SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N);
341 SDValue PromoteIntRes_VECREDUCE(SDNode *N);
342 SDValue PromoteIntRes_ABS(SDNode *N);
346 SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
347 SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
348 SDValue PromoteIntOp_BITCAST(SDNode *N);
349 SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
350 SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
351 SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);
352 SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);
353 SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
354 SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
355 SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
356 SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
357 SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N);
358 SDValue PromoteIntOp_SPLAT_VECTOR(SDNode *N);
359 SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
360 SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
361 SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
362 SDValue PromoteIntOp_Shift(SDNode *N);
363 SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
364 SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
365 SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
366 SDValue PromoteIntOp_TRUNCATE(SDNode *N);
367 SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
368 SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
369 SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
370 SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
371 SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
372 SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
373 SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
374 SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
375 SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
376 SDValue PromoteIntOp_MULFIX(SDNode *N);
377 SDValue PromoteIntOp_FPOWI(SDNode *N);
378 SDValue PromoteIntOp_VECREDUCE(SDNode *N);
380 void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
380 void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
392 void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
392 void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
392 void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
393 void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
393 void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
393 void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
397 void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
397 void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
398 void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
398 void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
399 void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
399 void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
400 void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
400 void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
401 void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
401 void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
402 void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
402 void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
403 void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
403 void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
404 void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
404 void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
405 void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
405 void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
406 void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);
406 void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);
407 void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
407 void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
408 void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
408 void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
409 void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);
409 void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);
410 void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
410 void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
411 void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi);
411 void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi);
412 void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi);
412 void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi);
413 void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi);
413 void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi);
414 void ExpandIntRes_LLROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
414 void ExpandIntRes_LLROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
415 void ExpandIntRes_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
415 void ExpandIntRes_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
417 void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);
417 void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);
418 void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
418 void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
419 void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);
419 void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);
420 void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);
420 void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);
421 void ExpandIntRes_ADDSUBCARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
421 void ExpandIntRes_ADDSUBCARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
422 void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);
422 void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);
423 void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);
423 void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);
424 void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);
424 void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);
425 void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
425 void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
426 void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);
426 void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);
427 void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
427 void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
428 void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
428 void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
429 void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);
429 void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);
431 void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);
431 void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);
433 void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
433 void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
434 void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
434 void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
435 void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
435 void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
436 void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
436 void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
437 void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
437 void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
439 void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
439 void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
440 void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi);
440 void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi);
443 SDValue &Lo, SDValue &Hi);
443 SDValue &Lo, SDValue &Hi);
444 bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
444 bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
445 bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
445 bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
449 SDValue ExpandIntOp_BR_CC(SDNode *N);
450 SDValue ExpandIntOp_SELECT_CC(SDNode *N);
451 SDValue ExpandIntOp_SETCC(SDNode *N);
452 SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
453 SDValue ExpandIntOp_Shift(SDNode *N);
454 SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
455 SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
456 SDValue ExpandIntOp_TRUNCATE(SDNode *N);
457 SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
458 SDValue ExpandIntOp_RETURNADDR(SDNode *N);
459 SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
461 void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
461 void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
473 SDValue GetSoftenedFloat(SDValue Op) {
473 SDValue GetSoftenedFloat(SDValue Op) {
481 SDValue SoftenedOp = getSDValue(Iter->second);
485 void SetSoftenedFloat(SDValue Op, SDValue Result);
485 void SetSoftenedFloat(SDValue Op, SDValue Result);
489 SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
490 SDValue SoftenFloatRes_BITCAST(SDNode *N);
491 SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
492 SDValue SoftenFloatRes_ConstantFP(SDNode *N);
493 SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);
494 SDValue SoftenFloatRes_FABS(SDNode *N);
495 SDValue SoftenFloatRes_FMINNUM(SDNode *N);
496 SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
497 SDValue SoftenFloatRes_FADD(SDNode *N);
498 SDValue SoftenFloatRes_FCBRT(SDNode *N);
499 SDValue SoftenFloatRes_FCEIL(SDNode *N);
500 SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N);
501 SDValue SoftenFloatRes_FCOS(SDNode *N);
502 SDValue SoftenFloatRes_FDIV(SDNode *N);
503 SDValue SoftenFloatRes_FEXP(SDNode *N);
504 SDValue SoftenFloatRes_FEXP2(SDNode *N);
505 SDValue SoftenFloatRes_FFLOOR(SDNode *N);
506 SDValue SoftenFloatRes_FLOG(SDNode *N);
507 SDValue SoftenFloatRes_FLOG2(SDNode *N);
508 SDValue SoftenFloatRes_FLOG10(SDNode *N);
509 SDValue SoftenFloatRes_FMA(SDNode *N);
510 SDValue SoftenFloatRes_FMUL(SDNode *N);
511 SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);
512 SDValue SoftenFloatRes_FNEG(SDNode *N);
513 SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);
514 SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);
515 SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
516 SDValue SoftenFloatRes_FPOW(SDNode *N);
517 SDValue SoftenFloatRes_FPOWI(SDNode *N);
518 SDValue SoftenFloatRes_FREM(SDNode *N);
519 SDValue SoftenFloatRes_FRINT(SDNode *N);
520 SDValue SoftenFloatRes_FROUND(SDNode *N);
521 SDValue SoftenFloatRes_FSIN(SDNode *N);
522 SDValue SoftenFloatRes_FSQRT(SDNode *N);
523 SDValue SoftenFloatRes_FSUB(SDNode *N);
524 SDValue SoftenFloatRes_FTRUNC(SDNode *N);
525 SDValue SoftenFloatRes_LOAD(SDNode *N);
526 SDValue SoftenFloatRes_SELECT(SDNode *N);
527 SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
528 SDValue SoftenFloatRes_UNDEF(SDNode *N);
529 SDValue SoftenFloatRes_VAARG(SDNode *N);
530 SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
534 SDValue SoftenFloatOp_BITCAST(SDNode *N);
535 SDValue SoftenFloatOp_BR_CC(SDNode *N);
536 SDValue SoftenFloatOp_FP_EXTEND(SDNode *N);
537 SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
538 SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
539 SDValue SoftenFloatOp_LROUND(SDNode *N);
540 SDValue SoftenFloatOp_LLROUND(SDNode *N);
541 SDValue SoftenFloatOp_LRINT(SDNode *N);
542 SDValue SoftenFloatOp_LLRINT(SDNode *N);
543 SDValue SoftenFloatOp_SELECT_CC(SDNode *N);
544 SDValue SoftenFloatOp_SETCC(SDNode *N);
545 SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
557 void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
557 void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
557 void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
558 void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
558 void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
558 void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
562 void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
562 void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
563 void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
563 void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
564 void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
564 void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
565 void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
565 void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
566 void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
566 void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
567 void ExpandFloatRes_FCBRT (SDNode *N, SDValue &Lo, SDValue &Hi);
567 void ExpandFloatRes_FCBRT (SDNode *N, SDValue &Lo, SDValue &Hi);
568 void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
568 void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
569 void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
569 void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
570 void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);
570 void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);
571 void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
571 void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
572 void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
572 void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
573 void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);
573 void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);
574 void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);
574 void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);
575 void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);
575 void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);
576 void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);
576 void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);
577 void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);
577 void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);
578 void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);
578 void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);
579 void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);
579 void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);
580 void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
580 void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
581 void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);
581 void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);
582 void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
582 void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
583 void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);
583 void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);
584 void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);
584 void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);
585 void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);
585 void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);
586 void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
586 void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
587 void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
587 void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
588 void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);
588 void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);
589 void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);
589 void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);
590 void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
590 void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
591 void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);
591 void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);
592 void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
592 void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
593 void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
593 void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
597 SDValue ExpandFloatOp_BR_CC(SDNode *N);
598 SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);
599 SDValue ExpandFloatOp_FP_ROUND(SDNode *N);
600 SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N);
601 SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N);
602 SDValue ExpandFloatOp_LROUND(SDNode *N);
603 SDValue ExpandFloatOp_LLROUND(SDNode *N);
604 SDValue ExpandFloatOp_LRINT(SDNode *N);
605 SDValue ExpandFloatOp_LLRINT(SDNode *N);
606 SDValue ExpandFloatOp_SELECT_CC(SDNode *N);
607 SDValue ExpandFloatOp_SETCC(SDNode *N);
608 SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);
610 void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
610 void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
617 SDValue GetPromotedFloat(SDValue Op) {
617 SDValue GetPromotedFloat(SDValue Op) {
619 SDValue PromotedOp = getSDValue(PromotedId);
623 void SetPromotedFloat(SDValue Op, SDValue Result);
623 void SetPromotedFloat(SDValue Op, SDValue Result);
626 SDValue PromoteFloatRes_BITCAST(SDNode *N);
627 SDValue PromoteFloatRes_BinOp(SDNode *N);
628 SDValue PromoteFloatRes_ConstantFP(SDNode *N);
629 SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
630 SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
631 SDValue PromoteFloatRes_FMAD(SDNode *N);
632 SDValue PromoteFloatRes_FPOWI(SDNode *N);
633 SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
634 SDValue PromoteFloatRes_LOAD(SDNode *N);
635 SDValue PromoteFloatRes_SELECT(SDNode *N);
636 SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
637 SDValue PromoteFloatRes_UnaryOp(SDNode *N);
638 SDValue PromoteFloatRes_UNDEF(SDNode *N);
639 SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
640 SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
643 SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
644 SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
645 SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
646 SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
647 SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
648 SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
649 SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
658 SDValue GetScalarizedVector(SDValue Op) {
658 SDValue GetScalarizedVector(SDValue Op) {
660 SDValue ScalarizedOp = getSDValue(ScalarizedId);
664 void SetScalarizedVector(SDValue Op, SDValue Result);
664 void SetScalarizedVector(SDValue Op, SDValue Result);
668 SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
669 SDValue ScalarizeVecRes_BinOp(SDNode *N);
670 SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
671 SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
672 SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
673 SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
674 SDValue ScalarizeVecRes_InregOp(SDNode *N);
675 SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
677 SDValue ScalarizeVecRes_BITCAST(SDNode *N);
678 SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
679 SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
680 SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
681 SDValue ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N);
682 SDValue ScalarizeVecRes_FPOWI(SDNode *N);
683 SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
684 SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
685 SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
686 SDValue ScalarizeVecRes_VSELECT(SDNode *N);
687 SDValue ScalarizeVecRes_SELECT(SDNode *N);
688 SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
689 SDValue ScalarizeVecRes_SETCC(SDNode *N);
690 SDValue ScalarizeVecRes_UNDEF(SDNode *N);
691 SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
693 SDValue ScalarizeVecRes_MULFIX(SDNode *N);
697 SDValue ScalarizeVecOp_BITCAST(SDNode *N);
698 SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
699 SDValue ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N);
700 SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
701 SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
702 SDValue ScalarizeVecOp_VSELECT(SDNode *N);
703 SDValue ScalarizeVecOp_VSETCC(SDNode *N);
704 SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
705 SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
706 SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);
707 SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
720 void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
720 void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
720 void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
721 void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
721 void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
721 void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
725 void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
725 void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
726 void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
726 void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
727 void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
727 void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
728 void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
728 void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
729 void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
729 void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
730 void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
730 void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
731 void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
731 void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
733 SDValue &Lo, SDValue &Hi);
733 SDValue &Lo, SDValue &Hi);
735 void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi);
735 void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi);
737 void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
737 void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
738 void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
738 void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
739 void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
739 void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
740 void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
740 void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
741 void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
741 void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
742 void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi);
742 void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi);
743 void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi);
743 void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi);
744 void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
744 void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
745 void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
745 void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
746 void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
746 void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
747 void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
747 void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
748 void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
748 void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
749 void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
749 void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
750 void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
751 SDValue &Hi);
752 void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);
752 void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);
756 SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
757 SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
758 SDValue SplitVecOp_UnaryOp(SDNode *N);
759 SDValue SplitVecOp_TruncateHelper(SDNode *N);
761 SDValue SplitVecOp_BITCAST(SDNode *N);
762 SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
763 SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
764 SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
765 SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
766 SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
767 SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
768 SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, unsigned OpNo);
769 SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
770 SDValue SplitVecOp_VSETCC(SDNode *N);
771 SDValue SplitVecOp_FP_ROUND(SDNode *N);
772 SDValue SplitVecOp_FCOPYSIGN(SDNode *N);
784 SDValue GetWidenedVector(SDValue Op) {
784 SDValue GetWidenedVector(SDValue Op) {
786 SDValue WidenedOp = getSDValue(WidenedId);
790 void SetWidenedVector(SDValue Op, SDValue Result);
790 void SetWidenedVector(SDValue Op, SDValue Result);
794 SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
795 SDValue WidenVecRes_BITCAST(SDNode* N);
796 SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
797 SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
798 SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
799 SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
800 SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
801 SDValue WidenVecRes_LOAD(SDNode* N);
802 SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
803 SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
804 SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
805 SDValue WidenVecRes_SELECT(SDNode* N);
806 SDValue WidenVSELECTAndMask(SDNode *N);
807 SDValue WidenVecRes_SELECT_CC(SDNode* N);
808 SDValue WidenVecRes_SETCC(SDNode* N);
809 SDValue WidenVecRes_UNDEF(SDNode *N);
810 SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
812 SDValue WidenVecRes_Ternary(SDNode *N);
813 SDValue WidenVecRes_Binary(SDNode *N);
814 SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
815 SDValue WidenVecRes_BinaryWithExtraScalarOp(SDNode *N);
816 SDValue WidenVecRes_StrictFP(SDNode *N);
817 SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);
818 SDValue WidenVecRes_Convert(SDNode *N);
819 SDValue WidenVecRes_Convert_StrictFP(SDNode *N);
820 SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
821 SDValue WidenVecRes_POWI(SDNode *N);
822 SDValue WidenVecRes_Shift(SDNode *N);
823 SDValue WidenVecRes_Unary(SDNode *N);
824 SDValue WidenVecRes_InregOp(SDNode *N);
828 SDValue WidenVecOp_BITCAST(SDNode *N);
829 SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
830 SDValue WidenVecOp_EXTEND(SDNode *N);
831 SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
832 SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
833 SDValue WidenVecOp_STORE(SDNode* N);
834 SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
835 SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
836 SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
837 SDValue WidenVecOp_SETCC(SDNode* N);
838 SDValue WidenVecOp_VSELECT(SDNode *N);
840 SDValue WidenVecOp_Convert(SDNode *N);
841 SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
842 SDValue WidenVecOp_VECREDUCE(SDNode *N);
847 SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE);
857 SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
857 SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
865 SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
865 SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
872 void GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST);
878 void GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
885 SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
885 SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
889 SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
889 SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
899 void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
899 void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
899 void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
910 void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
910 void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
910 void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
914 SDValue &Lo, SDValue &Hi);
914 SDValue &Lo, SDValue &Hi);
915 void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi);
915 void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi);
916 void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
916 void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
917 void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
917 void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
930 void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
930 void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
930 void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
940 void IntegerToVector(SDValue Op, unsigned NumElements,
941 SmallVectorImpl<SDValue> &Ops, EVT EltVT);
945 SDValue &Lo, SDValue &Hi);
945 SDValue &Lo, SDValue &Hi);
946 void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);
946 void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);
947 void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);
947 void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);
948 void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);
948 void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);
949 void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
949 void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
950 void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);
950 void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);
951 void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);
951 void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);
954 SDValue ExpandOp_BITCAST (SDNode *N);
955 SDValue ExpandOp_BUILD_VECTOR (SDNode *N);
956 SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N);
957 SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);
958 SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);
959 SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo);
lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp 35 SDValue &Lo, SDValue &Hi) {
35 SDValue &Lo, SDValue &Hi) {
36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
43 SDValue InOp = N->getOperand(0);
118 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
120 SmallVector<SDValue, 8> Vals;
131 SDValue LHS = Vals[Slot];
132 SDValue RHS = Vals[Slot + 1];
159 SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
165 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
186 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,
187 SDValue &Hi) {
193 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,
194 SDValue &Hi) {
196 SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ?
205 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
206 SDValue &Hi) {
207 SDValue OldVec = N->getOperand(0);
226 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
232 SDValue Idx = N->getOperand(1);
245 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
246 SDValue &Hi) {
253 SDValue Chain = LD->getChain();
254 SDValue Ptr = LD->getBasePtr();
286 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
286 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
289 SDValue Chain = N->getOperand(0);
290 SDValue Ptr = N->getOperand(1);
312 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
313 SmallVectorImpl<SDValue> &Ops,
317 SDValue Parts[2];
331 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {
356 SmallVector<SDValue, 8> Ops;
359 SDValue Vec =
368 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
381 SmallVector<SDValue, 16> NewElts;
385 SDValue Lo, Hi;
394 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
400 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {
401 SDValue Lo, Hi;
406 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
412 SDValue Val = N->getOperand(1);
422 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
425 SDValue Lo, Hi;
430 SDValue Idx = N->getOperand(2);
442 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
448 SmallVector<SDValue, 16> Ops(NumElts);
450 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
456 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
464 SDValue Chain = St->getChain();
465 SDValue Ptr = St->getBasePtr();
472 SDValue Lo, Hi;
501 SDValue &Lo, SDValue &Hi) {
501 SDValue &Lo, SDValue &Hi) {
502 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
506 void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
506 void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
507 SDValue LL, LH, RL, RH, CL, CH;
512 SDValue Cond = N->getOperand(0);
515 if (SDValue Res = WidenVSELECTAndMask(N))
543 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
544 SDValue &Hi) {
545 SDValue LL, LH, RL, RH;
556 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
556 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp 64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
67 void AddLegalizedOperand(SDValue From, SDValue To) {
67 void AddLegalizedOperand(SDValue From, SDValue To) {
75 SDValue LegalizeOp(SDValue Op);
75 SDValue LegalizeOp(SDValue Op);
78 SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
78 SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
78 SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
81 SDValue UnrollVSETCC(SDValue Op);
81 SDValue UnrollVSETCC(SDValue Op);
87 SDValue Expand(SDValue Op);
87 SDValue Expand(SDValue Op);
91 SDValue ExpandFP_TO_UINT(SDValue Op);
91 SDValue ExpandFP_TO_UINT(SDValue Op);
95 SDValue ExpandUINT_TO_FLOAT(SDValue Op);
95 SDValue ExpandUINT_TO_FLOAT(SDValue Op);
98 SDValue ExpandSEXTINREG(SDValue Op);
98 SDValue ExpandSEXTINREG(SDValue Op);
105 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op);
105 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op);
112 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op);
112 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op);
118 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op);
118 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op);
124 SDValue ExpandABS(SDValue Op);
124 SDValue ExpandABS(SDValue Op);
127 SDValue ExpandBSWAP(SDValue Op);
127 SDValue ExpandBSWAP(SDValue Op);
131 SDValue ExpandVSELECT(SDValue Op);
131 SDValue ExpandVSELECT(SDValue Op);
132 SDValue ExpandSELECT(SDValue Op);
132 SDValue ExpandSELECT(SDValue Op);
133 SDValue ExpandLoad(SDValue Op);
133 SDValue ExpandLoad(SDValue Op);
134 SDValue ExpandStore(SDValue Op);
134 SDValue ExpandStore(SDValue Op);
135 SDValue ExpandFNEG(SDValue Op);
135 SDValue ExpandFNEG(SDValue Op);
136 SDValue ExpandFSUB(SDValue Op);
136 SDValue ExpandFSUB(SDValue Op);
137 SDValue ExpandBITREVERSE(SDValue Op);
137 SDValue ExpandBITREVERSE(SDValue Op);
138 SDValue ExpandCTPOP(SDValue Op);
138 SDValue ExpandCTPOP(SDValue Op);
139 SDValue ExpandCTLZ(SDValue Op);
139 SDValue ExpandCTLZ(SDValue Op);
140 SDValue ExpandCTTZ(SDValue Op);
140 SDValue ExpandCTTZ(SDValue Op);
141 SDValue ExpandFunnelShift(SDValue Op);
141 SDValue ExpandFunnelShift(SDValue Op);
142 SDValue ExpandROT(SDValue Op);
142 SDValue ExpandROT(SDValue Op);
143 SDValue ExpandFMINNUM_FMAXNUM(SDValue Op);
143 SDValue ExpandFMINNUM_FMAXNUM(SDValue Op);
144 SDValue ExpandUADDSUBO(SDValue Op);
144 SDValue ExpandUADDSUBO(SDValue Op);
145 SDValue ExpandSADDSUBO(SDValue Op);
145 SDValue ExpandSADDSUBO(SDValue Op);
146 SDValue ExpandMULO(SDValue Op);
146 SDValue ExpandMULO(SDValue Op);
147 SDValue ExpandAddSubSat(SDValue Op);
147 SDValue ExpandAddSubSat(SDValue Op);
148 SDValue ExpandFixedPointMul(SDValue Op);
148 SDValue ExpandFixedPointMul(SDValue Op);
149 SDValue ExpandStrictFPOp(SDValue Op);
149 SDValue ExpandStrictFPOp(SDValue Op);
155 SDValue Promote(SDValue Op);
155 SDValue Promote(SDValue Op);
160 SDValue PromoteINT_TO_FP(SDValue Op);
160 SDValue PromoteINT_TO_FP(SDValue Op);
166 SDValue PromoteFP_TO_INT(SDValue Op);
166 SDValue PromoteFP_TO_INT(SDValue Op);
210 SDValue OldRoot = DAG.getRoot();
222 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) {
222 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) {
222 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) {
229 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
229 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
232 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
232 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
238 SmallVector<SDValue, 8> Ops;
239 for (const SDValue &Op : Node->op_values())
242 SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops),
257 if (SDValue Lowered = TLI.LowerOperation(Result, DAG)) {
285 SDValue Lowered = TLI.LowerOperation(Result, DAG);
303 for (const SDValue &Op : Node->op_values())
496 if (SDValue Tmp1 = TLI.LowerOperation(Op, DAG)) {
520 SDValue VectorLegalizer::Promote(SDValue Op) {
520 SDValue VectorLegalizer::Promote(SDValue Op) {
544 SmallVector<SDValue, 4> Operands(Op.getNumOperands());
569 SDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) {
569 SDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) {
578 SmallVector<SDValue, 4> Operands(Op.getNumOperands());
596 SDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op) {
596 SDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op) {
610 SDValue Promoted = DAG.getNode(NewOpc, dl, NVT, Op.getOperand(0));
622 SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
622 SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
629 SDValue NewChain;
630 SDValue Value;
634 SmallVector<SDValue, 8> Vals;
635 SmallVector<SDValue, 8> LoadChains;
638 SDValue Chain = LD->getChain();
639 SDValue BasePTR = LD->getBasePtr();
658 SmallVector<SDValue, 8> LoadVals;
660 SDValue ScalarLoad;
697 SDValue SrcEltBitMask = DAG.getConstant(
703 SDValue ShAmt = DAG.getConstant(
705 SDValue Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt);
715 SDValue Hi =
746 SDValue Scalarized = TLI.scalarizeVectorLoad(LD, DAG);
763 SDValue VectorLegalizer::ExpandStore(SDValue Op) {
763 SDValue VectorLegalizer::ExpandStore(SDValue Op) {
765 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
770 SDValue VectorLegalizer::Expand(SDValue Op) {
770 SDValue VectorLegalizer::Expand(SDValue Op) {
887 SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
887 SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
894 SDValue Mask = Op.getOperand(0);
895 SDValue Op1 = Op.getOperand(1);
896 SDValue Op2 = Op.getOperand(2);
932 SDValue AllOnes = DAG.getConstant(
934 SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes);
938 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
942 SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) {
942 SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) {
955 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
964 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) {
964 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) {
968 SDValue Src = Op.getOperand(0);
1000 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) {
1000 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) {
1003 SDValue Src = Op.getOperand(0);
1015 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1024 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) {
1024 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) {
1028 SDValue Src = Op.getOperand(0);
1046 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1071 SDValue VectorLegalizer::ExpandBSWAP(SDValue Op) {
1071 SDValue VectorLegalizer::ExpandBSWAP(SDValue Op) {
1089 SDValue VectorLegalizer::ExpandBITREVERSE(SDValue Op) {
1089 SDValue VectorLegalizer::ExpandBITREVERSE(SDValue Op) {
1132 SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
1132 SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
1137 SDValue Mask = Op.getOperand(0);
1138 SDValue Op1 = Op.getOperand(1);
1139 SDValue Op2 = Op.getOperand(2);
1169 SDValue AllOnes = DAG.getConstant(
1171 SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes);
1175 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1179 SDValue VectorLegalizer::ExpandABS(SDValue Op) {
1179 SDValue VectorLegalizer::ExpandABS(SDValue Op) {
1181 SDValue Result;
1189 SDValue VectorLegalizer::ExpandFP_TO_UINT(SDValue Op) {
1189 SDValue VectorLegalizer::ExpandFP_TO_UINT(SDValue Op) {
1191 SDValue Result, Chain;
1203 SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
1203 SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
1208 SDValue Result;
1221 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1227 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1230 SDValue TWOHW = DAG.getConstantFP(1ULL << (BW / 2), DL, Op.getValueType());
1233 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord);
1234 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), HalfWordMask);
1239 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), HI);
1241 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), LO);
1247 SDValue VectorLegalizer::ExpandFNEG(SDValue Op) {
1247 SDValue VectorLegalizer::ExpandFNEG(SDValue Op) {
1250 SDValue Zero = DAG.getConstantFP(-0.0, DL, Op.getValueType());
1258 SDValue VectorLegalizer::ExpandFSUB(SDValue Op) {
1258 SDValue VectorLegalizer::ExpandFSUB(SDValue Op) {
1270 SDValue VectorLegalizer::ExpandCTPOP(SDValue Op) {
1270 SDValue VectorLegalizer::ExpandCTPOP(SDValue Op) {
1271 SDValue Result;
1278 SDValue VectorLegalizer::ExpandCTLZ(SDValue Op) {
1278 SDValue VectorLegalizer::ExpandCTLZ(SDValue Op) {
1279 SDValue Result;
1286 SDValue VectorLegalizer::ExpandCTTZ(SDValue Op) {
1286 SDValue VectorLegalizer::ExpandCTTZ(SDValue Op) {
1287 SDValue Result;
1294 SDValue VectorLegalizer::ExpandFunnelShift(SDValue Op) {
1294 SDValue VectorLegalizer::ExpandFunnelShift(SDValue Op) {
1295 SDValue Result;
1302 SDValue VectorLegalizer::ExpandROT(SDValue Op) {
1302 SDValue VectorLegalizer::ExpandROT(SDValue Op) {
1303 SDValue Result;
1310 SDValue VectorLegalizer::ExpandFMINNUM_FMAXNUM(SDValue Op) {
1310 SDValue VectorLegalizer::ExpandFMINNUM_FMAXNUM(SDValue Op) {
1311 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Op.getNode(), DAG))
1316 SDValue VectorLegalizer::ExpandUADDSUBO(SDValue Op) {
1316 SDValue VectorLegalizer::ExpandUADDSUBO(SDValue Op) {
1317 SDValue Result, Overflow;
1329 SDValue VectorLegalizer::ExpandSADDSUBO(SDValue Op) {
1329 SDValue VectorLegalizer::ExpandSADDSUBO(SDValue Op) {
1330 SDValue Result, Overflow;
1342 SDValue VectorLegalizer::ExpandMULO(SDValue Op) {
1342 SDValue VectorLegalizer::ExpandMULO(SDValue Op) {
1343 SDValue Result, Overflow;
1356 SDValue VectorLegalizer::ExpandAddSubSat(SDValue Op) {
1356 SDValue VectorLegalizer::ExpandAddSubSat(SDValue Op) {
1357 if (SDValue Expanded = TLI.expandAddSubSat(Op.getNode(), DAG))
1362 SDValue VectorLegalizer::ExpandFixedPointMul(SDValue Op) {
1362 SDValue VectorLegalizer::ExpandFixedPointMul(SDValue Op) {
1363 if (SDValue Expanded = TLI.expandFixedPointMul(Op.getNode(), DAG))
1368 SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) {
1368 SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) {
1375 SDValue Chain = Op.getOperand(0);
1378 SmallVector<SDValue, 32> OpValues;
1379 SmallVector<SDValue, 32> OpChains;
1381 SmallVector<SDValue, 4> Opers;
1382 SDValue Idx = DAG.getConstant(i, dl,
1390 SDValue Oper = Op.getOperand(j);
1400 SDValue ScalarOp = DAG.getNode(Op->getOpcode(), dl, ValueVTs, Opers);
1406 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1407 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1415 SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
1415 SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
1419 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1), CC = Op.getOperand(2);
1422 SmallVector<SDValue, 8> Ops(NumElems);
1424 SDValue LHSElem = DAG.getNode(
1427 SDValue RHSElem = DAG.getNode(
lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp 37 SDValue R = SDValue();
199 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
200 SDValue LHS = GetScalarizedVector(N->getOperand(0));
201 SDValue RHS = GetScalarizedVector(N->getOperand(1));
206 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
207 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
208 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
209 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
214 SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) {
215 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
216 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
217 SDValue Op2 = N->getOperand(2);
222 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
225 SDValue Chain = N->getOperand(0);
229 SmallVector<SDValue, 4> Opers;
236 SDValue Oper = N->getOperand(i);
244 SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
252 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
258 SDValue ScalarLHS, ScalarRHS;
263 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
281 SDValue OtherVal = DAG.getNode(
289 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
291 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
295 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
296 SDValue Op = N->getOperand(0);
306 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
308 SDValue InOp = N->getOperand(0);
316 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
322 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
324 SDValue Op = GetScalarizedVector(N->getOperand(0));
329 SDValue DAGTypeLegalizer::ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N) {
331 SDValue Op = GetScalarizedVector(N->getOperand(1));
332 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
341 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
342 SDValue Op = GetScalarizedVector(N->getOperand(0));
347 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
350 SDValue Op = N->getOperand(1);
358 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
361 SDValue Result = DAG.getLoad(
375 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
378 SDValue Op = N->getOperand(0);
399 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
402 SDValue LHS = GetScalarizedVector(N->getOperand(0));
407 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
409 SDValue Op = N->getOperand(0);
435 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
439 SDValue InOp = N->getOperand(0);
445 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
446 SDValue Cond = N->getOperand(0);
461 SDValue LHS = GetScalarizedVector(N->getOperand(1));
514 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
515 SDValue LHS = GetScalarizedVector(N->getOperand(1));
521 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
522 SDValue LHS = GetScalarizedVector(N->getOperand(2));
529 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
533 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
535 SDValue Arg = N->getOperand(2).getOperand(0);
542 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
546 SDValue LHS = N->getOperand(0);
547 SDValue RHS = N->getOperand(1);
567 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
584 SDValue Res = SDValue();
669 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
670 SDValue Elt = GetScalarizedVector(N->getOperand(0));
677 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
680 SDValue Elt = GetScalarizedVector(N->getOperand(0));
681 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
690 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
693 SDValue Elt = GetScalarizedVector(N->getOperand(1));
694 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
706 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
707 SmallVector<SDValue, 8> Ops(N->getNumOperands());
715 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
717 SDValue Res = GetScalarizedVector(N->getOperand(0));
728 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
729 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
739 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
746 SDValue LHS = GetScalarizedVector(N->getOperand(0));
747 SDValue RHS = GetScalarizedVector(N->getOperand(1));
753 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
768 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
788 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
789 SDValue Elt = GetScalarizedVector(N->getOperand(0));
790 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
796 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
799 SDValue Elt = GetScalarizedVector(N->getOperand(1));
800 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
810 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
811 SDValue Res = GetScalarizedVector(N->getOperand(0));
828 SDValue Lo, Hi;
1014 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
1015 SDValue &Hi) {
1016 SDValue LHSLo, LHSHi;
1018 SDValue RHSLo, RHSHi;
1028 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1029 SDValue &Hi) {
1030 SDValue Op0Lo, Op0Hi;
1032 SDValue Op1Lo, Op1Hi;
1034 SDValue Op2Lo, Op2Hi;
1044 void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1044 void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1045 SDValue LHSLo, LHSHi;
1047 SDValue RHSLo, RHSHi;
1050 SDValue Op2 = N->getOperand(2);
1057 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1058 SDValue &Hi) {
1065 SDValue InOp = N->getOperand(0);
1114 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1115 SDValue &Hi) {
1120 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1123 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1127 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1128 SDValue &Hi) {
1141 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1144 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1148 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1149 SDValue &Hi) {
1150 SDValue Vec = N->getOperand(0);
1151 SDValue Idx = N->getOperand(1);
1164 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1165 SDValue &Hi) {
1166 SDValue Vec = N->getOperand(0);
1167 SDValue SubVec = N->getOperand(1);
1168 SDValue Idx = N->getOperand(2);
1193 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1194 SDValue Store =
1198 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1218 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1219 SDValue &Hi) {
1226 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1227 SDValue &Hi) {
1228 SDValue LHSLo, LHSHi;
1232 SDValue RHSLo, RHSHi;
1233 SDValue RHS = N->getOperand(1);
1245 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1246 SDValue &Hi) {
1247 SDValue LHSLo, LHSHi;
1261 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1262 SDValue &Hi) {
1264 SDValue N0 = N->getOperand(0);
1267 SDValue InLo, InHi;
1299 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1300 SDValue &Hi) {
1302 SDValue Chain = N->getOperand(0);
1307 SmallVector<SDValue, 4> OpsLo;
1308 SmallVector<SDValue, 4> OpsHi;
1316 SDValue Op = N->getOperand(i);
1317 SDValue OpLo = Op;
1318 SDValue OpHi = Op;
1349 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1350 SDValue Chain = N->getOperand(0);
1356 SmallVector<SDValue, 8> Scalars;
1357 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1367 SmallVector<SDValue, 8> Chains;
1373 SDValue Operand = N->getOperand(j);
1385 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1407 SDValue &Lo, SDValue &Hi) {
1407 SDValue &Lo, SDValue &Hi) {
1415 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1440 SDValue OtherVal = DAG.getNode(
1447 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1448 SDValue &Hi) {
1449 SDValue Vec = N->getOperand(0);
1450 SDValue Elt = N->getOperand(1);
1451 SDValue Idx = N->getOperand(2);
1487 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1491 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1495 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1526 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1527 SDValue &Hi) {
1535 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1536 SDValue &Hi) {
1543 SDValue Ch = LD->getChain();
1544 SDValue Ptr = LD->getBasePtr();
1545 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1574 SDValue &Lo, SDValue &Hi) {
1574 SDValue &Lo, SDValue &Hi) {
1579 SDValue Ch = MLD->getChain();
1580 SDValue Ptr = MLD->getBasePtr();
1581 SDValue Mask = MLD->getMask();
1582 SDValue PassThru = MLD->getPassThru();
1587 SDValue MaskLo, MaskHi;
1601 SDValue PassThruLo, PassThruHi;
1639 SDValue &Lo, SDValue &Hi) {
1639 SDValue &Lo, SDValue &Hi) {
1644 SDValue Ch = MGT->getChain();
1645 SDValue Ptr = MGT->getBasePtr();
1646 SDValue Mask = MGT->getMask();
1647 SDValue PassThru = MGT->getPassThru();
1648 SDValue Index = MGT->getIndex();
1649 SDValue Scale = MGT->getScale();
1653 SDValue MaskLo, MaskHi;
1668 SDValue PassThruLo, PassThruHi;
1674 SDValue IndexHi, IndexLo;
1685 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1689 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1704 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1704 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1714 SDValue LL, LH, RL, RH;
1731 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1732 SDValue &Hi) {
1755 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1765 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1774 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1775 SDValue &Hi) {
1809 SDValue NewSrc =
1824 SDValue &Lo, SDValue &Hi) {
1824 SDValue &Lo, SDValue &Hi) {
1826 SDValue Inputs[4];
1838 SDValue &Output = High ? Hi : Lo;
1889 SmallVector<SDValue, 16> SVOps;
1920 SDValue Op0 = Inputs[InputUsed[0]];
1922 SDValue Op1 = InputUsed[1] == -1U ?
1932 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1932 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1935 SDValue Chain = N->getOperand(0);
1936 SDValue Ptr = N->getOperand(1);
1937 SDValue SV = N->getOperand(2);
1963 SDValue Res = SDValue();
2073 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2078 SDValue Mask = N->getOperand(0);
2079 SDValue Src0 = N->getOperand(1);
2080 SDValue Src1 = N->getOperand(2);
2085 SDValue Lo, Hi;
2094 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2099 SDValue LoSelect =
2101 SDValue HiSelect =
2107 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2109 SDValue Lo, Hi;
2112 SDValue VecOp = N->getOperand(OpNo);
2145 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2149 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2152 SDValue Lo, Hi;
2168 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2182 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2186 SDValue Lo, Hi;
2198 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2201 SDValue Idx = N->getOperand(1);
2203 SDValue Lo, Hi;
2220 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2221 SDValue Vec = N->getOperand(0);
2222 SDValue Idx = N->getOperand(1);
2228 SDValue Lo, Hi;
2255 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2259 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2267 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2277 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2278 SDValue Lo, Hi;
2287 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2293 SDValue Ch = MGT->getChain();
2294 SDValue Ptr = MGT->getBasePtr();
2295 SDValue Index = MGT->getIndex();
2296 SDValue Scale = MGT->getScale();
2297 SDValue Mask = MGT->getMask();
2298 SDValue PassThru = MGT->getPassThru();
2301 SDValue MaskLo, MaskHi;
2312 SDValue PassThruLo, PassThruHi;
2318 SDValue IndexHi, IndexLo;
2329 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2330 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2339 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2340 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2352 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2358 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2360 SDValue Ch = N->getChain();
2361 SDValue Ptr = N->getBasePtr();
2362 SDValue Mask = N->getMask();
2363 SDValue Data = N->getValue();
2371 SDValue DataLo, DataHi;
2379 SDValue MaskLo, MaskHi;
2389 SDValue Lo, Hi;
2416 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2418 SDValue Ch = N->getChain();
2419 SDValue Ptr = N->getBasePtr();
2420 SDValue Mask = N->getMask();
2421 SDValue Index = N->getIndex();
2422 SDValue Scale = N->getScale();
2423 SDValue Data = N->getValue();
2432 SDValue DataLo, DataHi;
2440 SDValue MaskLo, MaskHi;
2450 SDValue IndexHi, IndexLo;
2456 SDValue Lo;
2462 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2474 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2479 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2485 SDValue Ch = N->getChain();
2486 SDValue Ptr = N->getBasePtr();
2491 SDValue Lo, Hi;
2525 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2533 SmallVector<SDValue, 32> Elts;
2535 for (const SDValue &Op : N->op_values()) {
2547 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2565 SDValue InVec = N->getOperand(0);
2600 SDValue InLoVec, InHiVec;
2609 SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2610 SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2613 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2626 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2631 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2641 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2650 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2653 SDValue Lo, Hi;
2668 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2679 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2698 SDValue Res = SDValue();
2907 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2911 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2912 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2913 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2917 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2921 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2922 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2926 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
2930 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2931 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2932 SDValue InOp3 = N->getOperand(2);
2940 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
2941 SmallVectorImpl<SDValue> &ConcatOps,
2973 SDValue VecOp = DAG.getUNDEF(NextVT);
2984 SDValue undefVec = DAG.getUNDEF(VT);
2986 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3010 SDValue UndefVal = DAG.getUNDEF(MaxVT);
3018 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3034 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3035 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3045 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3046 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3049 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3060 SDValue EOp1 = DAG.getNode(
3063 SDValue EOp2 = DAG.getNode(
3077 SDValue EOp1 = DAG.getNode(
3080 SDValue EOp2 = DAG.getNode(
3093 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3113 SmallVector<SDValue, 4> InOps;
3116 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3117 SmallVector<SDValue, 16> Chains;
3126 SDValue Oper = N->getOperand(i);
3144 SmallVector<SDValue, 4> EOps;
3147 SDValue Op = InOps[i];
3158 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3171 SmallVector<SDValue, 4> EOps;
3174 SDValue Op = InOps[i];
3186 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3195 SDValue NewChain;
3205 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3210 SDValue WideLHS, WideRHS;
3227 SDValue Zero = DAG.getConstant(
3247 SDValue Zero = DAG.getConstant(
3249 SDValue OtherVal = DAG.getNode(
3257 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3258 SDValue InOp = N->getOperand(0);
3302 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3304 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3311 SDValue InVal = DAG.getNode(
3323 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3328 SDValue Val = DAG.getNode(
3340 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3341 SDValue InOp = N->getOperand(1);
3343 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3359 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3360 SmallVector<SDValue, 32> OpChains;
3371 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3377 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3379 SDValue InOp = N->getOperand(0);
3404 SmallVector<SDValue, 16> Ops;
3406 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3430 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3441 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3443 SDValue InOp = GetWidenedVector(N->getOperand(0));
3444 SDValue ShOp = N->getOperand(1);
3448 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3450 SDValue InOp = GetWidenedVector(N->getOperand(0));
3451 SDValue ShOp = N->getOperand(1);
3467 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3470 SDValue InOp = GetWidenedVector(N->getOperand(0));
3474 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3480 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3485 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3486 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3490 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3491 SDValue InOp = N->getOperand(0);
3550 SDValue NewVec;
3557 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3571 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3584 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3591 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3605 SDValue UndefVal = DAG.getUNDEF(InVT);
3606 SmallVector<SDValue, 16> Ops(NumConcat);
3644 SmallVector<SDValue, 16> Ops(WidenNumElts);
3647 SDValue InOp = N->getOperand(i);
3655 SDValue UndefVal = DAG.getUNDEF(EltVT);
3661 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3665 SDValue InOp = N->getOperand(0);
3666 SDValue Idx = N->getOperand(1);
3686 SmallVector<SDValue, 16> Ops(WidenNumElts);
3696 SDValue UndefVal = DAG.getUNDEF(EltVT);
3702 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3703 SDValue InOp = GetWidenedVector(N->getOperand(0));
3709 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3713 SDValue Result;
3714 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3723 SDValue NewChain;
3736 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3739 SDValue Mask = N->getMask();
3741 SDValue PassThru = GetWidenedVector(N->getPassThru());
3751 SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3761 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3764 SDValue Mask = N->getMask();
3766 SDValue PassThru = GetWidenedVector(N->getPassThru());
3767 SDValue Scale = N->getScale();
3778 SDValue Index = N->getIndex();
3783 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3785 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3795 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3815 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3841 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3841 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3849 SmallVector<SDValue, 4> Ops;
3852 SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3877 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3883 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3898 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3900 SDValue Cond = N->getOperand(0);
3946 SDValue VSelOp1 = N->getOperand(1);
3947 SDValue VSelOp2 = N->getOperand(2);
3959 SDValue Mask;
3967 SDValue SETCC0 = Cond->getOperand(0);
3968 SDValue SETCC1 = Cond->getOperand(1);
4004 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4008 SDValue Cond1 = N->getOperand(0);
4011 if (SDValue Res = WidenVSELECTAndMask(N))
4026 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4027 SDValue Res = ModifyToType(SplitSelect, WidenVT);
4035 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4036 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4042 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4043 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4044 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4050 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4055 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4063 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4064 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4080 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4087 SDValue InOp1 = N->getOperand(0);
4097 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4098 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4103 SDValue InOp2 = N->getOperand(1);
4129 SDValue Res = SDValue();
4211 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4215 SDValue InOp = N->getOperand(0);
4272 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4279 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4285 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4298 SDValue Res;
4315 SmallVector<SDValue, 16> Ops(NumElts);
4317 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4318 SmallVector<SDValue, 32> OpChains;
4326 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4340 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4342 SDValue InOp = GetWidenedVector(N->getOperand(0));
4354 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4372 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4382 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4403 SmallVector<SDValue, 16> Ops(NumElts);
4409 SDValue InOp = N->getOperand(i);
4422 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4423 SDValue InOp = GetWidenedVector(N->getOperand(0));
4428 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4429 SDValue InOp = GetWidenedVector(N->getOperand(0));
4434 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4442 SmallVector<SDValue, 16> StChain;
4454 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4458 SDValue Mask = MST->getMask();
4460 SDValue StVal = MST->getValue();
4493 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4496 SDValue DataOp = MG->getPassThru();
4497 SDValue Mask = MG->getMask();
4498 SDValue Scale = MG->getScale();
4501 SDValue Index = GetWidenedVector(MG->getIndex());
4504 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4506 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4513 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4515 SDValue DataOp = MSC->getValue();
4516 SDValue Mask = MSC->getMask();
4517 SDValue Index = MSC->getIndex();
4518 SDValue Scale = MSC->getScale();
4541 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4548 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4549 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4550 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4566 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4573 SDValue CC = DAG.getNode(
4583 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4585 SDValue Op = GetWidenedVector(N->getOperand(0));
4590 SDValue NeutralElem;
4639 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4646 SDValue Cond = GetWidenedVector(N->getOperand(0));
4647 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4648 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4651 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4730 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4731 SmallVectorImpl<SDValue> &LdOps,
4741 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4760 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4760 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4774 SDValue Chain = LD->getChain();
4775 SDValue BasePtr = LD->getBasePtr();
4787 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4796 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4804 SmallVector<SDValue, 16> ConcatOps(NumConcat);
4805 SDValue UndefVal = DAG.getUNDEF(NewVT);
4813 SmallVector<SDValue, 16> LdOps;
4824 SDValue L;
4837 SmallVector<SDValue, 16> Loads;
4868 SmallVector<SDValue, 16> ConcatOps(End);
4900 SmallVector<SDValue, 16> WidenOps(NumOps);
4901 SDValue UndefVal = DAG.getUNDEF(LdTy);
4912 SDValue
4913 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4924 SDValue Chain = LD->getChain();
4925 SDValue BasePtr = LD->getBasePtr();
4936 SmallVector<SDValue, 16> Ops(WidenNumElts);
4944 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4952 SDValue UndefVal = DAG.getUNDEF(EltVT);
4959 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
4964 SDValue Chain = ST->getChain();
4965 SDValue BasePtr = ST->getBasePtr();
4969 SDValue ValOp = GetWidenedVector(ST->getValue());
4990 SDValue EOp = DAG.getNode(
5006 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5010 SDValue EOp = DAG.getNode(
5028 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
5032 SDValue Chain = ST->getChain();
5033 SDValue BasePtr = ST->getBasePtr();
5037 SDValue ValOp = GetWidenedVector(ST->getValue());
5054 SDValue EOp = DAG.getNode(
5062 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
5063 SDValue EOp = DAG.getNode(
5075 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5075 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5092 SmallVector<SDValue, 16> Ops(NumConcat);
5093 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5108 SmallVector<SDValue, 16> Ops(WidenNumElts);
5117 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp 130 const SDValue &Op = ScegN->getOperand(i);
335 const SDValue &Op = SU->getNode()->getOperand(i);
485 const SDValue &Op = ScegN->getOperand(i);
lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp 227 for (const SDValue &Op : N->op_values()) {
677 const SDValue &Op = N->getOperand(NumLeft-1);
764 DenseMap<SDValue, unsigned> VRBaseMap;
lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp 451 for (const SDValue &Op : N->op_values())
467 for (const SDValue &Op : N->op_values())
498 for (const SDValue &Op : N->op_values()) {
525 for (const SDValue &Op : N->op_values())
1161 for (const SDValue &Op : N->op_values()) {
1335 for (const SDValue &Op : N->op_values())
1420 const SDValue &OptionalDef = Node->getOperand(i - Node->getNumValues());
lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp 141 SDValue ExtraOper = SDValue()) {
142 SmallVector<SDValue, 8> Ops(N->op_begin(), N->op_end());
161 static bool AddGlue(SDNode *N, SDValue Glue, bool AddGlue, SelectionDAG *DAG) {
286 SDValue InGlue = SDValue(nullptr, 0);
350 for (const SDValue &Op : NI->op_values())
380 SDValue GlueVal(N, N->getNumValues()-1);
734 DenseMap<SDValue, unsigned> &VRBaseMap, unsigned Order) {
761 DenseMap<SDValue, unsigned> &VRBaseMap,
829 DenseMap<SDValue, unsigned> VRBaseMap;
lib/CodeGen/SelectionDAG/SelectionDAG.cpp 102 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
176 SDValue NotZero = N->getOperand(i);
204 for (const SDValue &Op : N->op_values()) {
237 for (const SDValue &Op : N->op_values()) {
250 for (const SDValue &Op : N->op_values()) {
268 bool ISD::matchUnaryPredicate(SDValue Op,
295 SDValue LHS, SDValue RHS,
295 SDValue LHS, SDValue RHS,
313 SDValue LHSOp = LHS.getOperand(i);
314 SDValue RHSOp = RHS.getOperand(i);
448 ArrayRef<SDValue> Ops) {
465 SDVTList VTList, ArrayRef<SDValue> OpList) {
935 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
940 SDValue Ops[] = { Op };
955 SDValue Op1, SDValue Op2,
955 SDValue Op1, SDValue Op2,
960 SDValue Ops[] = { Op1, Op2 };
974 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1098 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1098 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1104 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1104 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1110 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1110 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1116 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1116 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1122 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1122 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1131 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1131 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1143 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1143 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1149 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1149 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1156 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1156 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1158 SDValue NegOne =
1163 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1163 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1164 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1168 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1183 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1192 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1197 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1234 SmallVector<SDValue, 2> EltParts;
1253 SmallVector<SDValue, 8> Ops;
1257 SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1281 SDValue Result(N, 0);
1290 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1295 SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1301 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1306 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1331 SDValue Result(N, 0);
1338 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1356 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1389 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1404 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1423 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1451 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1477 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1494 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1508 SDValue SelectionDAG::getValueType(EVT VT) {
1522 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1530 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1539 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1549 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1564 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1564 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1569 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1569 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1570 SDValue N2, ArrayRef<int> Mask) {
1606 SDValue Splat = BV->getSplatValue(&UndefElements);
1670 SDValue V = N1;
1680 SDValue Splat = BV->getSplatValue(&UndefElements);
1703 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1704 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1716 SDValue Ops[2] = { N1, N2 };
1737 SDValue V = SDValue(N, 0);
1742 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1747 SDValue Op0 = SV.getOperand(0);
1748 SDValue Op1 = SV.getOperand(1);
1752 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1767 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1781 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1781 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1786 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1787 SDValue Root, MCSymbol *Label) {
1789 SDValue Ops[] = { Root };
1805 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1825 SDValue SelectionDAG::getSrcValue(const Value *V) {
1843 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1858 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1858 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1865 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1865 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1867 SDValue Ops[] = {Ptr};
1888 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1888 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1896 SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1901 SDValue Tmp1 = Node->getOperand(0);
1902 SDValue Tmp2 = Node->getOperand(1);
1905 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1907 SDValue VAList = VAListLoad;
1930 SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1937 SDValue Tmp1 =
1944 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1955 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1968 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
1968 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
1968 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2127 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2127 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2140 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2140 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2171 if (SDValue SimplifyLHS =
2190 SDValue Src = V.getOperand(0);
2198 if (SDValue DemandedSrc = GetDemandedBits(Src, SrcDemandedBits))
2208 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2216 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2228 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2235 bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2242 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2256 SDValue Scl;
2258 SDValue Op = V.getOperand(i);
2291 SDValue Src = V.getOperand(0);
2310 SDValue LHS = V.getOperand(0);
2311 SDValue RHS = V.getOperand(1);
2325 bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2336 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2336 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2373 SDValue SelectionDAG::getSplatValue(SDValue V) {
2373 SDValue SelectionDAG::getSplatValue(SDValue V) {
2375 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx))
2384 static const APInt *getValidShiftAmountConstant(SDValue V) {
2397 static const APInt *getValidMinimumShiftAmountConstant(SDValue V) {
2421 KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2432 KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2472 SDValue SrcOp = Op.getOperand(i);
2519 SDValue LHS = Op.getOperand(0);
2528 SDValue RHS = Op.getOperand(1);
2545 SDValue Sub = Op.getOperand(i);
2559 SDValue Src = Op.getOperand(0);
2560 SDValue Sub = Op.getOperand(1);
2593 SDValue Src = Op.getOperand(0);
2611 SDValue N0 = Op.getOperand(0);
2619 SDValue N0 = Op.getOperand(0);
3152 SDValue InVec = Op.getOperand(0);
3153 SDValue EltNo = Op.getOperand(1);
3176 SDValue InVec = Op.getOperand(0);
3177 SDValue InVal = Op.getOperand(1);
3178 SDValue EltNo = Op.getOperand(2);
3330 SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3331 SDValue N1) const {
3361 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3402 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3410 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3446 SDValue SrcOp = Op.getOperand(i);
3494 SDValue N0 = Op.getOperand(0);
3549 SDValue Src = Op.getOperand(0);
3760 SDValue InVec = Op.getOperand(0);
3761 SDValue InVal = Op.getOperand(1);
3762 SDValue EltNo = Op.getOperand(2);
3797 SDValue InVec = Op.getOperand(0);
3798 SDValue EltNo = Op.getOperand(1);
3823 SDValue Src = Op.getOperand(0);
3855 SDValue Src = Op.getOperand(0);
3856 SDValue Sub = Op.getOperand(1);
3968 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
3980 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
4097 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4108 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4130 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4130 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4145 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4145 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4151 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4152 ArrayRef<SDValue> Ops,
4164 SDValue IdentitySrc;
4185 static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4186 ArrayRef<SDValue> Ops,
4208 SDValue IdentitySrc;
4211 SDValue Op = Ops[i];
4234 SmallVector<SDValue, 16> Elts;
4235 for (SDValue Op : Ops) {
4247 for (SDValue Op : Elts)
4251 for (SDValue &Op : Elts)
4256 SDValue V = DAG.getBuildVector(VT, DL, Elts);
4262 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4274 SDValue V = SDValue(N, 0);
4279 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4280 SDValue Operand, const SDNodeFlags Flags) {
4450 SDValue Ops = { Operand };
4451 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4466 SDValue Ops[] = {Operand};
4467 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4557 SDValue OpOp = Operand.getOperand(0);
4669 SDValue Ops[] = {Operand};
4689 SDValue V = SDValue(N, 0);
4736 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4749 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
4769 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
4778 SDValue Divisor = Ops[1];
4794 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4808 SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, C1, C2);
4844 SmallVector<SDValue, 4> Outputs;
4847 SDValue V1 = BV1 ? BV1->getOperand(I) : getUNDEF(SVT);
4848 SDValue V2 = BV2 ? BV2->getOperand(I) : getUNDEF(SVT);
4860 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4882 SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4884 ArrayRef<SDValue> Ops,
4933 SmallVector<SDValue, 4> ScalarResults;
4935 SmallVector<SDValue, 4> ScalarOps;
4936 for (SDValue Op : Ops) {
4948 SDValue ScalarOp = InBV->getOperand(i);
4960 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
4973 SDValue V = getBuildVector(VT, DL, ScalarResults);
4978 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
4979 EVT VT, SDValue N1, SDValue N2) {
4979 EVT VT, SDValue N1, SDValue N2) {
5036 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5037 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5037 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5066 SDValue Ops[] = {N1, N2};
5067 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5072 SDValue Ops[] = {N1, N2};
5073 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5127 if (SDValue V = simplifyFPBinop(Opcode, N1, N2))
5139 if (SDValue V = simplifyShift(N1, N2))
5212 SmallVector<SDValue, 8> Ops;
5215 SDValue Op = N1.getOperand(i);
5257 SDValue Elt = N1.getOperand(N2C->getZExtValue());
5274 SDValue N1Op2 = N1.getOperand(2);
5362 if (SDValue SV =
5366 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
5420 SDValue Ops[] = {N1, N2};
5440 SDValue V = SDValue(N, 0);
5445 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5446 SDValue N1, SDValue N2, SDValue N3,
5446 SDValue N1, SDValue N2, SDValue N3,
5446 SDValue N1, SDValue N2, SDValue N3,
5468 SDValue Ops[] = {N1, N2, N3};
5469 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5474 SDValue Ops[] = {N1, N2, N3};
5475 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5489 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
5492 SDValue Ops[] = {N1, N2, N3};
5493 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
5501 if (SDValue V = simplifySelect(N1, N2, N3))
5526 SDValue Index = N3;
5565 SDValue Ops[] = {N1, N2, N3};
5585 SDValue V = SDValue(N, 0);
5590 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5591 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5591 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5591 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5591 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5592 SDValue Ops[] = { N1, N2, N3, N4 };
5596 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5597 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5597 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5597 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5597 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5598 SDValue N5) {
5599 SDValue Ops[] = { N1, N2, N3, N4, N5 };
5605 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
5605 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
5606 SmallVector<SDValue, 8> ArgChains;
5627 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
5627 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
5669 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
5711 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
5711 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
5718 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
5745 SmallVector<SDValue, 32> &OutChains, unsigned From,
5746 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
5747 SmallVector<SDValue, 16> &OutStoreChains) {
5750 SmallVector<SDValue, 16> GluedLoadChains;
5757 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5762 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
5769 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
5770 SDValue Chain, SDValue Dst, SDValue Src,
5770 SDValue Chain, SDValue Dst, SDValue Src,
5770 SDValue Chain, SDValue Dst, SDValue Src,
5833 SmallVector<SDValue, 16> OutLoadChains;
5834 SmallVector<SDValue, 16> OutStoreChains;
5835 SmallVector<SDValue, 32> OutChains;
5841 SDValue Value, Store;
5954 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
5955 SDValue Chain, SDValue Dst, SDValue Src,
5955 SDValue Chain, SDValue Dst, SDValue Src,
5955 SDValue Chain, SDValue Dst, SDValue Src,
6007 SmallVector<SDValue, 8> LoadValues;
6008 SmallVector<SDValue, 8> LoadChains;
6009 SmallVector<SDValue, 8> OutChains;
6014 SDValue Value;
6034 SDValue Store;
6064 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
6065 SDValue Chain, SDValue Dst, SDValue Src,
6065 SDValue Chain, SDValue Dst, SDValue Src,
6065 SDValue Chain, SDValue Dst, SDValue Src,
6105 SmallVector<SDValue, 8> OutChains;
6114 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
6128 SDValue Value = MemSetValue;
6137 SDValue Store = DAG.getStore(
6159 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6159 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6159 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6160 SDValue Src, SDValue Size, unsigned Align,
6160 SDValue Src, SDValue Size, unsigned Align,
6174 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
6184 SDValue Result = TSI->EmitTargetCodeForMemcpy(
6230 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6230 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6234 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
6234 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
6235 SDValue Dst, unsigned DstAlign,
6236 SDValue Src, unsigned SrcAlign,
6237 SDValue Size, Type *SizeTy,
6271 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6271 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6275 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6275 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6275 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6276 SDValue Src, SDValue Size, unsigned Align,
6276 SDValue Src, SDValue Size, unsigned Align,
6290 SDValue Result =
6301 SDValue Result = TSI->EmitTargetCodeForMemmove(
6334 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6334 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6338 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
6338 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
6339 SDValue Dst, unsigned DstAlign,
6340 SDValue Src, unsigned SrcAlign,
6341 SDValue Size, Type *SizeTy,
6375 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6375 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6379 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
6379 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
6379 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
6380 SDValue Src, SDValue Size, unsigned Align,
6380 SDValue Src, SDValue Size, unsigned Align,
6393 SDValue Result =
6404 SDValue Result = TSI->EmitTargetCodeForMemset(
6436 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6436 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6440 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
6440 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
6441 SDValue Dst, unsigned DstAlign,
6442 SDValue Value, SDValue Size, Type *SizeTy,
6442 SDValue Value, SDValue Size, Type *SizeTy,
6476 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6476 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6480 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6481 SDVTList VTList, ArrayRef<SDValue> Ops,
6502 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
6503 EVT MemVT, SDVTList VTs, SDValue Chain,
6504 SDValue Ptr, SDValue Cmp, SDValue Swp,
6504 SDValue Ptr, SDValue Cmp, SDValue Swp,
6504 SDValue Ptr, SDValue Cmp, SDValue Swp,
6510 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
6514 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6515 SDValue Chain, SDValue Ptr, SDValue Val,
6515 SDValue Chain, SDValue Ptr, SDValue Val,
6515 SDValue Chain, SDValue Ptr, SDValue Val,
6538 SDValue Ops[] = {Chain, Ptr, Val};
6542 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6543 EVT VT, SDValue Chain, SDValue Ptr,
6543 EVT VT, SDValue Chain, SDValue Ptr,
6548 SDValue Ops[] = {Chain, Ptr};
6553 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
6553 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
6564 SDValue SelectionDAG::getMemIntrinsicNode(
6565 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
6581 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
6583 ArrayRef<SDValue> Ops, EVT MemVT,
6619 SDValue V(N, 0);
6624 SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
6625 SDValue Chain, int FrameIndex,
6629 SDValue Ops[2] = {
6649 SDValue V(N, 0);
6659 SelectionDAG &DAG, SDValue Ptr,
6683 SelectionDAG &DAG, SDValue Ptr,
6684 SDValue OffsetOp) {
6693 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6694 EVT VT, const SDLoc &dl, SDValue Chain,
6695 SDValue Ptr, SDValue Offset,
6695 SDValue Ptr, SDValue Offset,
6718 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6719 EVT VT, const SDLoc &dl, SDValue Chain,
6720 SDValue Ptr, SDValue Offset, EVT MemVT,
6720 SDValue Ptr, SDValue Offset, EVT MemVT,
6744 SDValue Ops[] = { Chain, Ptr, Offset };
6762 SDValue V(N, 0);
6767 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6767 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6768 SDValue Ptr, MachinePointerInfo PtrInfo,
6772 SDValue Undef = getUNDEF(Ptr.getValueType());
6777 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6777 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6778 SDValue Ptr, MachineMemOperand *MMO) {
6779 SDValue Undef = getUNDEF(Ptr.getValueType());
6784 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
6785 EVT VT, SDValue Chain, SDValue Ptr,
6785 EVT VT, SDValue Chain, SDValue Ptr,
6790 SDValue Undef = getUNDEF(Ptr.getValueType());
6795 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
6796 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
6796 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
6798 SDValue Undef = getUNDEF(Ptr.getValueType());
6803 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
6803 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
6804 SDValue Base, SDValue Offset,
6804 SDValue Base, SDValue Offset,
6818 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6818 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6818 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6819 SDValue Ptr, MachinePointerInfo PtrInfo,
6839 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6839 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6839 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6840 SDValue Ptr, MachineMemOperand *MMO) {
6845 SDValue Undef = getUNDEF(Ptr.getValueType());
6846 SDValue Ops[] = { Chain, Val, Ptr, Undef };
6864 SDValue V(N, 0);
6869 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6869 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6869 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6870 SDValue Ptr, MachinePointerInfo PtrInfo,
6891 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6891 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6891 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6892 SDValue Ptr, EVT SVT,
6912 SDValue Undef = getUNDEF(Ptr.getValueType());
6913 SDValue Ops[] = { Chain, Val, Ptr, Undef };
6931 SDValue V(N, 0);
6936 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
6936 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
6937 SDValue Base, SDValue Offset,
6937 SDValue Base, SDValue Offset,
6942 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
6959 SDValue V(N, 0);
6964 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6964 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6965 SDValue Ptr, SDValue Mask, SDValue PassThru,
6965 SDValue Ptr, SDValue Mask, SDValue PassThru,
6965 SDValue Ptr, SDValue Mask, SDValue PassThru,
6969 SDValue Ops[] = { Chain, Ptr, Mask, PassThru };
6987 SDValue V(N, 0);
6992 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
6992 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
6993 SDValue Val, SDValue Ptr, SDValue Mask,
6993 SDValue Val, SDValue Ptr, SDValue Mask,
6993 SDValue Val, SDValue Ptr, SDValue Mask,
6999 SDValue Ops[] = { Chain, Val, Ptr, Mask };
7017 SDValue V(N, 0);
7022 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
7023 ArrayRef<SDValue> Ops,
7058 SDValue V(N, 0);
7063 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
7064 ArrayRef<SDValue> Ops,
7096 SDValue V(N, 0);
7101 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7101 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7101 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7101 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7131 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7131 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7131 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7156 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y) {
7156 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y) {
7156 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y) {
7180 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
7180 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
7181 SDValue Ptr, SDValue SV, unsigned Align) {
7181 SDValue Ptr, SDValue SV, unsigned Align) {
7182 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
7186 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7198 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
7202 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7203 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
7217 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7221 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7262 SDValue V(N, 0);
7267 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
7268 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
7272 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7273 ArrayRef<SDValue> Ops) {
7317 SDValue V(N, 0);
7322 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
7327 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7328 SDValue N1) {
7329 SDValue Ops[] = { N1 };
7333 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7334 SDValue N1, SDValue N2) {
7334 SDValue N1, SDValue N2) {
7335 SDValue Ops[] = { N1, N2 };
7339 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7340 SDValue N1, SDValue N2, SDValue N3) {
7340 SDValue N1, SDValue N2, SDValue N3) {
7340 SDValue N1, SDValue N2, SDValue N3) {
7341 SDValue Ops[] = { N1, N2, N3 };
7345 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7346 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7346 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7346 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7346 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7347 SDValue Ops[] = { N1, N2, N3, N4 };
7351 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7352 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7352 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7352 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7352 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7353 SDValue N5) {
7354 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7448 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
7473 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
7473 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
7503 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
7503 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
7503 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
7504 SDValue Ops[] = { Op1, Op2, Op3 };
7509 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7509 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7510 SDValue Op3, SDValue Op4) {
7510 SDValue Op3, SDValue Op4) {
7511 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
7516 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7516 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7517 SDValue Op3, SDValue Op4, SDValue Op5) {
7517 SDValue Op3, SDValue Op4, SDValue Op5) {
7517 SDValue Op3, SDValue Op4, SDValue Op5) {
7518 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
7523 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
7595 EVT VT, SDValue Op1) {
7597 SDValue Ops[] = { Op1 };
7602 EVT VT, SDValue Op1,
7603 SDValue Op2) {
7605 SDValue Ops[] = { Op1, Op2 };
7610 EVT VT, SDValue Op1,
7611 SDValue Op2, SDValue Op3) {
7611 SDValue Op2, SDValue Op3) {
7613 SDValue Ops[] = { Op1, Op2, Op3 };
7618 EVT VT, ArrayRef<SDValue> Ops) {
7624 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
7637 ArrayRef<SDValue> Ops) {
7644 SDValue Op1, SDValue Op2) {
7644 SDValue Op1, SDValue Op2) {
7646 SDValue Ops[] = { Op1, Op2 };
7651 SDVTList VTs,ArrayRef<SDValue> Ops) {
7696 SDVTList VTs, ArrayRef<SDValue> Ops) {
7791 SDValue InputChain = Node->getOperand(0);
7792 SDValue OutputChain = SDValue(Node, 1);
7795 SmallVector<SDValue, 3> Ops;
7830 EVT VT, SDValue Op1) {
7832 SDValue Ops[] = { Op1 };
7837 EVT VT, SDValue Op1, SDValue Op2) {
7837 EVT VT, SDValue Op1, SDValue Op2) {
7839 SDValue Ops[] = { Op1, Op2 };
7844 EVT VT, SDValue Op1, SDValue Op2,
7844 EVT VT, SDValue Op1, SDValue Op2,
7845 SDValue Op3) {
7847 SDValue Ops[] = { Op1, Op2, Op3 };
7852 EVT VT, ArrayRef<SDValue> Ops) {
7858 EVT VT1, EVT VT2, SDValue Op1,
7859 SDValue Op2) {
7861 SDValue Ops[] = { Op1, Op2 };
7866 EVT VT1, EVT VT2, SDValue Op1,
7867 SDValue Op2, SDValue Op3) {
7867 SDValue Op2, SDValue Op3) {
7869 SDValue Ops[] = { Op1, Op2, Op3 };
7875 ArrayRef<SDValue> Ops) {
7882 SDValue Op1, SDValue Op2) {
7882 SDValue Op1, SDValue Op2) {
7884 SDValue Ops[] = { Op1, Op2 };
7890 SDValue Op1, SDValue Op2,
7890 SDValue Op1, SDValue Op2,
7891 SDValue Op3) {
7893 SDValue Ops[] = { Op1, Op2, Op3 };
7899 ArrayRef<SDValue> Ops) {
7906 ArrayRef<SDValue> Ops) {
7913 ArrayRef<SDValue> Ops) {
7941 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
7942 SDValue Operand) {
7943 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
7951 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
7952 SDValue Operand, SDValue Subreg) {
7952 SDValue Operand, SDValue Subreg) {
7953 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
7962 ArrayRef<SDValue> Ops,
8021 void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
8021 void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
8093 SDValue N0 = N.getOperand(0);
8094 SDValue N1 = N.getOperand(1);
8160 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
8160 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
8267 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
8292 const SDValue &ToOp = To[Use.getResNo()];
8314 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
8314 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
8461 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
8462 const SDValue *To,
8621 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
8622 SDValue NewMemOp) {
8628 SDValue OldChain = SDValue(OldLoad, 1);
8629 SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
8633 SDValue TokenFactor =
8640 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
8640 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
8670 bool llvm::isNullConstant(SDValue V) {
8675 bool llvm::isNullFPConstant(SDValue V) {
8680 bool llvm::isAllOnesConstant(SDValue V) {
8685 bool llvm::isOneConstant(SDValue V) {
8690 SDValue llvm::peekThroughBitcasts(SDValue V) {
8690 SDValue llvm::peekThroughBitcasts(SDValue V) {
8696 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
8696 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
8702 SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
8702 SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
8708 bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
8718 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
8741 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
8765 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
8779 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
8796 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
8802 bool llvm::isOneOrOneSplat(SDValue N) {
8809 bool llvm::isAllOnesOrAllOnesSplat(SDValue N) {
8965 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
9014 SDValue
9024 SDValue Op = Extract->getOperand(0);
9075 SDValue PrevOp;
9082 SDValue Op0 = Op.getOperand(0);
9083 SDValue Op1 = Op.getOperand(1);
9110 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
9119 SmallVector<SDValue, 8> Scalars;
9120 SmallVector<SDValue, 4> Operands(N->getNumOperands());
9131 SDValue Operand = N->getOperand(j);
9179 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
9179 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
9200 SmallVector<SDValue, 8> LHSScalars;
9201 SmallVector<SDValue, 8> RHSScalars;
9207 SmallVector<SDValue, 8> ResScalars;
9208 SmallVector<SDValue, 8> OvScalars;
9210 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
9211 SDValue Ov =
9257 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
9309 std::pair<SDValue, SDValue>
9309 std::pair<SDValue, SDValue>
9310 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
9315 SDValue Lo, Hi;
9325 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
9325 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
9333 void SelectionDAG::ExtractVectorElements(SDValue Op,
9334 SmallVectorImpl<SDValue> &Args,
9386 SDValue OpVal = getOperand(i);
9426 SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
9436 SDValue Splatted;
9440 SDValue Op = getOperand(i);
9461 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
9509 for (const SDValue &Op : op_values()) {
9538 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
9552 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
9562 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
9584 SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
9585 SmallVectorImpl<SDValue> &Vals) {
9590 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
9615 for (const SDValue &Op : N->op_values())
lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp 169 SDValue Ptr = N->getBasePtr();
172 SDValue Base = DAG.getTargetLoweringInfo().unwrapAddress(Ptr);
173 SDValue Index = SDValue();
245 SDValue PotentialBase = Base->getOperand(0);
lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp 190 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
191 const SDValue *Parts, unsigned NumParts,
200 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
201 const SDValue *Parts, unsigned NumParts,
211 SDValue Val = Parts[0];
225 SDValue Lo, Hi;
268 SDValue Lo, Hi;
357 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
358 const SDValue *Parts, unsigned NumParts,
366 SDValue Val = Parts[0];
393 SmallVector<SDValue, 8> Ops(NumIntermediates);
491 SDValue Val, SDValue *Parts, unsigned NumParts,
491 SDValue Val, SDValue *Parts, unsigned NumParts,
498 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
499 SDValue *Parts, unsigned NumParts, MVT PartVT,
585 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
611 SDValue &Part0 = Parts[i];
612 SDValue &Part1 = Parts[i+StepSize/2];
630 static SDValue widenVectorToPartType(SelectionDAG &DAG,
631 SDValue Val, const SDLoc &DL, EVT PartVT) {
643 SmallVector<SDValue, 16> Ops;
645 SDValue EltUndef = DAG.getUNDEF(ElementVT);
659 SDValue Val, SDValue *Parts, unsigned NumParts,
659 SDValue Val, SDValue *Parts, unsigned NumParts,
674 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
732 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
739 SmallVector<SDValue, 8> Ops(NumIntermediates);
799 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
801 const SDLoc &dl, SDValue &Chain,
802 SDValue *Flag, const Value *V) const {
810 SmallVector<SDValue, 4> Values(ValueVTs.size());
811 SmallVector<SDValue, 8> Parts;
823 SDValue P;
886 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
887 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
887 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
895 SmallVector<SDValue, 8> Parts(NumRegs);
913 SmallVector<SDValue, 8> Chains(NumRegs);
915 SDValue Part;
945 std::vector<SDValue> &Ops) const {
962 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1034 SDValue SelectionDAGBuilder::getRoot() {
1039 SDValue Root = PendingLoads[0];
1046 SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
1052 SDValue SelectionDAGBuilder::getControlRoot() {
1053 SDValue Root = DAG.getRoot();
1160 SDValue Val) {
1297 SDValue N = NodeMap[V];
1370 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1372 SDValue Result;
1380 SDValue Chain = DAG.getEntryNode();
1390 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1394 SDValue &N = NodeMap[V];
1399 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1403 SDValue Val = getValueImpl(V);
1417 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1419 SDValue &N = NodeMap[V];
1432 SDValue Val = getValueImpl(V);
1440 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1466 SDValue N1 = NodeMap[V];
1472 SmallVector<SDValue, 4> Constants;
1489 SmallVector<SDValue, 4> Ops;
1512 SmallVector<SDValue, 4> Constants(NumElts);
1534 SmallVector<SDValue, 16> Ops;
1543 SDValue Op;
1571 SDValue Chain = DAG.getEntryNode();
1628 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1755 SDValue Ret =
1767 SDValue Chain = getControlRoot();
1769 SmallVector<SDValue, 8> OutVals;
1796 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1798 SDValue RetOp = getValue(I.getOperand(0));
1806 SmallVector<SDValue, 4> Chains(NumValues);
1810 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1812 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1827 SDValue RetOp = getValue(I.getOperand(0));
1857 SmallVector<SDValue, 4> Parts(NumParts);
2312 SDValue Cond;
2313 SDValue CondLHS = getValue(CB.CmpLHS);
2339 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2342 SDValue CondRHS = getValue(CB.CmpRHS);
2359 SDValue CmpOp = getValue(CB.CmpMHS);
2366 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2385 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2389 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2407 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2409 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2410 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2424 SDValue SwitchOp = getValue(JTH.SValue);
2426 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2439 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2447 SDValue CMP = DAG.getSetCC(
2452 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2474 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2475 SDValue &Chain) {
2513 SDValue Guard;
2515 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2520 SDValue GuardVal = DAG.getLoad(
2550 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2550 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2557 SDValue Chain = DAG.getEntryNode();
2562 SDValue GuardPtr = getValue(IRGuard);
2571 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2573 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2580 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2584 SDValue Br = DAG.getNode(ISD::BR, dl,
2604 SDValue Chain =
2623 SDValue SwitchOp = getValue(B.SValue);
2625 SDValue RangeSub =
2642 SDValue Sub = RangeSub;
2650 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2659 SDValue Root = CopyTo;
2662 SDValue RangeCmp = DAG.getSetCC(dl,
2688 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2689 SDValue Cmp;
2707 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2711 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2727 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2776 SmallVector<SDValue, 8> Ops;
2885 SDValue Ops[2];
2902 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2966 SDValue Op2 = getValue(I.getOperand(1));
3105 SDValue Op = getValue(I.getOperand(0));
3106 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3125 SDValue Op1 = getValue(I.getOperand(0));
3126 SDValue Op2 = getValue(I.getOperand(1));
3127 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3133 SDValue Op1 = getValue(I.getOperand(0));
3134 SDValue Op2 = getValue(I.getOperand(1));
3180 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3186 SDValue Op1 = getValue(I.getOperand(0));
3187 SDValue Op2 = getValue(I.getOperand(1));
3202 SDValue Op1 = getValue(I.getOperand(0));
3203 SDValue Op2 = getValue(I.getOperand(1));
3229 SDValue Op1 = getValue(I.getOperand(0));
3230 SDValue Op2 = getValue(I.getOperand(1));
3257 SmallVector<SDValue, 4> Values(NumValues);
3258 SDValue Cond = getValue(I.getOperand(0));
3259 SDValue LHSVal = getValue(I.getOperand(1));
3260 SDValue RHSVal = getValue(I.getOperand(2));
3365 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3380 SDValue N = getValue(I.getOperand(0));
3389 SDValue N = getValue(I.getOperand(0));
3398 SDValue N = getValue(I.getOperand(0));
3406 SDValue N = getValue(I.getOperand(0));
3417 SDValue N = getValue(I.getOperand(0));
3425 SDValue N = getValue(I.getOperand(0));
3433 SDValue N = getValue(I.getOperand(0));
3441 SDValue N = getValue(I.getOperand(0));
3449 SDValue N = getValue(I.getOperand(0));
3458 SDValue N = getValue(I.getOperand(0));
3472 SDValue N = getValue(I.getOperand(0));
3482 SDValue N = getValue(I.getOperand(0));
3506 SDValue N = getValue(SV);
3520 SDValue InVec = getValue(I.getOperand(0));
3521 SDValue InVal = getValue(I.getOperand(1));
3522 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3531 SDValue InVec = getValue(I.getOperand(0));
3532 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3540 SDValue Src1 = getValue(I.getOperand(0));
3541 SDValue Src2 = getValue(I.getOperand(1));
3551 SDValue FirstElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3604 SmallVector<SDValue, 8> ConcatOps;
3624 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3626 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3627 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3643 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3690 SDValue &Src = Input == 0 ? Src1 : Src2;
3720 SmallVector<SDValue,8> Ops;
3722 SDValue Res;
3727 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3764 SmallVector<SDValue, 4> Values(NumAggValues);
3772 SDValue Agg = getValue(Op0);
3780 SDValue Val = getValue(Op1);
3820 SmallVector<SDValue, 4> Values(NumValValues);
3822 SDValue Agg = getValue(Op0);
3839 SDValue N = getValue(Op0);
3890 SDValue OffsVal = VectorWidth ?
3907 SDValue IdxN = getValue(Idx);
3927 SDValue Scale = DAG.getConstant(ElementSize.getZExtValue(), dl,
3959 SDValue AllocSize = getValue(I.getArraySize());
3990 SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3992 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4019 SDValue Ptr = getValue(SV);
4041 SDValue Root;
4069 SmallVector<SDValue, 4> Values(NumValues);
4070 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4082 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4087 SDValue A = DAG.getNode(ISD::ADD, dl,
4102 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4114 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4138 SDValue Src = getValue(SrcV);
4144 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4177 SDValue L = DAG.getCopyFromReg(
4217 SDValue Src = getValue(SrcV);
4218 SDValue Ptr = getValue(PtrV);
4220 SDValue Root = getRoot();
4221 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4244 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4249 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
4251 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4254 SDValue St =
4260 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4293 SDValue Ptr = getValue(PtrOperand);
4294 SDValue Src0 = getValue(Src0Operand);
4295 SDValue Mask = getValue(MaskOperand);
4309 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
4331 static bool getUniformBase(const Value *&Ptr, SDValue &Base, SDValue &Index,
4331 static bool getUniformBase(const Value *&Ptr, SDValue &Base, SDValue &Index,
4332 ISD::MemIndexType &IndexType, SDValue &Scale,
4389 SDValue Src0 = getValue(I.getArgOperand(0));
4390 SDValue Mask = getValue(I.getArgOperand(3));
4400 SDValue Base;
4401 SDValue Index;
4403 SDValue Scale;
4419 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4420 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4453 SDValue Ptr = getValue(PtrOperand);
4454 SDValue Src0 = getValue(Src0Operand);
4455 SDValue Mask = getValue(MaskOperand);
4475 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4483 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4495 SDValue Src0 = getValue(I.getArgOperand(3));
4496 SDValue Mask = getValue(I.getArgOperand(2));
4508 SDValue Root = DAG.getRoot();
4509 SDValue Base;
4510 SDValue Index;
4512 SDValue Scale;
4540 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4541 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4544 SDValue OutChain = Gather.getValue(1);
4556 SDValue InChain = getRoot();
4575 SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4581 SDValue OutChain = L.getValue(2);
4609 SDValue InChain = getRoot();
4625 SDValue L =
4630 SDValue OutChain = L.getValue(1);
4639 SDValue Ops[3];
4653 SDValue InChain = getRoot();
4684 SDValue Ptr = getValue(I.getPointerOperand());
4689 SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
4694 SDValue OutChain = L.getValue(1);
4702 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4705 SDValue OutChain = L.getValue(1);
4719 SDValue InChain = getRoot();
4739 SDValue Val = getValue(I.getValueOperand());
4742 SDValue Ptr = getValue(I.getPointerOperand());
4747 SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
4751 SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4770 SmallVector<SDValue, 8> Ops;
4822 SDValue Result;
4840 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4864 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4864 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4865 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4867 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4877 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4877 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4879 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4881 SDValue t1 = DAG.getNode(
4884 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4890 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4896 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4896 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4901 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4904 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4905 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4913 SDValue TwoToFractionalPartOfX;
4922 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4924 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4926 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4938 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4940 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4942 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4943 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4945 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4959 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4961 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4963 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4964 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4966 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4967 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4969 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4970 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4972 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4973 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4975 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4981 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4988 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4988 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4999 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5010 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5010 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5016 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5019 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5020 SDValue LogOfExponent =
5026 SDValue X = GetSignificand(DAG, Op1, dl);
5028 SDValue LogOfMantissa;
5037 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5039 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5041 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5054 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5056 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5058 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5059 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5061 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5062 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5064 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5079 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5081 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5083 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5084 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5086 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5087 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5089 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5090 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5092 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5093 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5095 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5109 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5109 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5115 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5118 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5122 SDValue X = GetSignificand(DAG, Op1, dl);
5126 SDValue Log2ofMantissa;
5133 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5135 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5137 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5150 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5152 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5154 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5155 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5157 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5158 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5160 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5176 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5178 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5180 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5181 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5183 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5184 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5186 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5187 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5189 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5190 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5192 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5206 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5206 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5212 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5215 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5216 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5221 SDValue X = GetSignificand(DAG, Op1, dl);
5223 SDValue Log10ofMantissa;
5232 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5234 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5236 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5248 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5250 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5252 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5253 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5255 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5269 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5271 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5273 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5274 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5276 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5277 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5279 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5280 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5282 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5296 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5296 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5308 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5308 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5308 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5326 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5336 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5336 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5336 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5360 SDValue Res; // Logically starts equal to 1.0
5361 SDValue CurSquare = LHS;
5393 const SDValue &N) {
5396 SDValue Op = N.getOperand(1);
5410 for (SDValue Op : N->op_values())
5423 DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
5524 SDValue LCandidate = peekThroughBitcasts(N);
5587 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5632 SDValue Callee = DAG.getExternalSymbol(
5644 SDValue Res;
5674 SDValue Chain = getRoot();
5675 SDValue RegName =
5687 SDValue Chain = getRoot();
5688 SDValue RegName =
5702 SDValue Op1 = getValue(I.getArgOperand(0));
5703 SDValue Op2 = getValue(I.getArgOperand(1));
5704 SDValue Op3 = getValue(I.getArgOperand(2));
5713 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5722 SDValue Op1 = getValue(I.getArgOperand(0));
5723 SDValue Op2 = getValue(I.getArgOperand(1));
5724 SDValue Op3 = getValue(I.getArgOperand(2));
5729 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5736 SDValue Op1 = getValue(I.getArgOperand(0));
5737 SDValue Op2 = getValue(I.getArgOperand(1));
5738 SDValue Op3 = getValue(I.getArgOperand(2));
5747 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5755 SDValue Dst = getValue(MI.getRawDest());
5756 SDValue Src = getValue(MI.getRawSource());
5757 SDValue Length = getValue(MI.getLength());
5764 SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5773 SDValue Dst = getValue(MI.getRawDest());
5774 SDValue Src = getValue(MI.getRawSource());
5775 SDValue Length = getValue(MI.getLength());
5782 SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5791 SDValue Dst = getValue(MI.getRawDest());
5792 SDValue Val = getValue(MI.getValue());
5793 SDValue Length = getValue(MI.getLength());
5799 SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5850 SDValue &N = NodeMap[Address];
5964 SDValue Ops[2];
5967 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6154 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
6158 SDValue Add = DAG.getNode(ISD::FADD, sdl,
6180 SDValue Tmp = getValue(I.getArgOperand(0));
6185 SDValue Op = getRoot();
6203 SDValue Arg = getValue(I.getArgOperand(0));
6211 SDValue Arg = getValue(I.getArgOperand(0));
6219 SDValue Arg = getValue(I.getArgOperand(0));
6227 SDValue X = getValue(I.getArgOperand(0));
6228 SDValue Y = getValue(I.getArgOperand(1));
6229 SDValue Z = getValue(I.getArgOperand(2));
6231 SDValue BitWidthC = DAG.getConstant(VT.getScalarSizeInBits(), sdl, VT);
6232 SDValue Zero = DAG.getConstant(0, sdl, VT);
6233 SDValue ShAmt = DAG.getNode(ISD::UREM, sdl, VT, Z, BitWidthC);
6255 SDValue NegShAmt = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
6262 SDValue NegZ = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
6263 SDValue NShAmt = DAG.getNode(ISD::UREM, sdl, VT, NegZ, BitWidthC);
6264 SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : NShAmt);
6265 SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, X, IsFSHL ? NShAmt : ShAmt);
6272 SDValue InvShAmt = DAG.getNode(ISD::SUB, sdl, VT, BitWidthC, ShAmt);
6273 SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : InvShAmt);
6274 SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, Y, IsFSHL ? InvShAmt : ShAmt);
6275 SDValue Or = DAG.getNode(ISD::OR, sdl, VT, ShX, ShY);
6285 SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ);
6290 SDValue Op1 = getValue(I.getArgOperand(0));
6291 SDValue Op2 = getValue(I.getArgOperand(1));
6296 SDValue Op1 = getValue(I.getArgOperand(0));
6297 SDValue Op2 = getValue(I.getArgOperand(1));
6302 SDValue Op1 = getValue(I.getArgOperand(0));
6303 SDValue Op2 = getValue(I.getArgOperand(1));
6308 SDValue Op1 = getValue(I.getArgOperand(0));
6309 SDValue Op2 = getValue(I.getArgOperand(1));
6315 SDValue Op1 = getValue(I.getArgOperand(0));
6316 SDValue Op2 = getValue(I.getArgOperand(1));
6317 SDValue Op3 = getValue(I.getArgOperand(2));
6323 SDValue Op1 = getValue(I.getArgOperand(0));
6324 SDValue Op2 = getValue(I.getArgOperand(1));
6325 SDValue Op3 = getValue(I.getArgOperand(2));
6331 SDValue Op1 = getValue(I.getArgOperand(0));
6332 SDValue Op2 = getValue(I.getArgOperand(1));
6333 SDValue Op3 = getValue(I.getArgOperand(2));
6339 SDValue Op = getRoot();
6352 SDValue Op = getRoot();
6370 SDValue Chain = getRoot();
6391 SDValue Src, Chain = getRoot();
6403 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
6447 SDValue Ops[6];
6509 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6509 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6530 SDValue Op1 = getValue(I.getArgOperand(0));
6531 SDValue Op2 = getValue(I.getArgOperand(1));
6544 SDValue Ops[5];
6552 SDValue Result = DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
6681 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6682 SDValue OffsetVal =
6687 SDValue FPVal = getValue(FP);
6688 SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
6701 SDValue N =
6717 SmallVector<SDValue, 8> Ops;
6720 SDValue LogEntryVal = getValue(I.getArgOperand(0));
6721 SDValue StrSizeVal = getValue(I.getArgOperand(1));
6723 SDValue Chain = getRoot();
6734 SDValue patchableNode = SDValue(MN, 0);
6748 SmallVector<SDValue, 8> Ops;
6753 SDValue LogTypeId = getValue(I.getArgOperand(0));
6754 SDValue LogEntryVal = getValue(I.getArgOperand(1));
6755 SDValue StrSizeVal = getValue(I.getArgOperand(2));
6757 SDValue Chain = getRoot();
6769 SDValue patchableNode = SDValue(MN, 0);
6795 SmallVector<SDValue, 16> Ops;
6808 SDValue Target;
6819 SDValue Val = getValue(I.getArgOperand(Op + 1));
6839 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL,
6858 SDValue Val = TSI.EmitTargetCodeForSetTag(
6867 SDValue Ptr = getValue(I.getOperand(0));
6868 SDValue Const = getValue(I.getOperand(1));
6984 SDValue Chain = getRoot();
6990 SDValue Result;
7017 SDValue OutChain = Result.getValue(1);
7019 SDValue FPResult = Result.getValue(0);
7023 std::pair<SDValue, SDValue>
7023 std::pair<SDValue, SDValue>
7054 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7054 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7096 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
7126 SDValue ArgNode = getValue(V);
7154 SDValue ArgNode = getValue(V);
7177 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7177 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7190 SDValue Src = CLI.InVals.back();
7193 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
7198 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
7219 SDValue Root;
7231 SDValue Ptr = Builder.getValue(PtrVal);
7232 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
7244 SDValue Value,
7272 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
7272 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
7332 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
7333 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
7342 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
7358 std::pair<SDValue, SDValue> Res =
7358 std::pair<SDValue, SDValue> Res =
7377 SDValue Dst = getValue(I.getArgOperand(0));
7378 SDValue Src = getValue(I.getArgOperand(1));
7379 SDValue Size = getValue(I.getArgOperand(2));
7393 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
7405 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
7420 std::pair<SDValue, SDValue> Res =
7420 std::pair<SDValue, SDValue> Res =
7443 std::pair<SDValue, SDValue> Res =
7443 std::pair<SDValue, SDValue> Res =
7466 std::pair<SDValue, SDValue> Res =
7466 std::pair<SDValue, SDValue> Res =
7487 std::pair<SDValue, SDValue> Res =
7487 std::pair<SDValue, SDValue> Res =
7511 SDValue Tmp = getValue(I.getArgOperand(0));
7527 SDValue Tmp0 = getValue(I.getArgOperand(0));
7528 SDValue Tmp1 = getValue(I.getArgOperand(1));
7570 SDValue LHS = getValue(I.getArgOperand(0));
7571 SDValue RHS = getValue(I.getArgOperand(1));
7708 SDValue Callee = getValue(I.getCalledValue());
7728 SDValue CallOperand;
7835 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
7835 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
7864 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
7986 const std::vector<SDValue> &AsmNodeOperands) {
8126 SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
8175 std::vector<SDValue> AsmNodeOperands;
8244 SDValue InOperandVal = OpInfo.CallOperand;
8313 std::vector<SDValue> Ops;
8412 SmallVector<SDValue, 1> ResultValues;
8413 SmallVector<SDValue, 8> OutChains;
8456 SDValue Val;
8482 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
8489 for (const SDValue &V : Val->op_values())
8504 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
8531 SmallVector<SDValue, 1> Ops;
8548 SDValue V = DAG.getVAArg(
8576 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
8578 SDValue Op) {
8599 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
8605 SmallVector<SDValue, 4> Ops;
8622 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
8667 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
8670 SDValue OpVal = Builder.getValue(CS.getArgument(i));
8692 SDValue Chain, InFlag, Callee, NullPtr;
8693 SmallVector<SDValue, 32> Ops;
8713 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
8716 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
8762 SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
8774 SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
8791 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8791 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8805 SmallVector<SDValue, 8> Ops;
8808 SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
8811 SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
8888 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
8889 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
8902 SDValue Op1 = getValue(I.getArgOperand(0));
8903 SDValue Op2;
8908 SDValue Res;
8986 std::pair<SDValue, SDValue>
8986 std::pair<SDValue, SDValue>
9022 SDValue DemoteStackSlot;
9124 SDValue Op = SDValue(Args[i].Node.getNode(),
9200 SmallVector<SDValue, 4> Parts(NumParts);
9258 SmallVector<SDValue, 4> InVals;
9289 SmallVector<SDValue, 4> ReturnValues;
9302 SmallVector<SDValue, 4> Chains(NumValues);
9310 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
9313 SDValue L = CLI.DAG.getLoad(
9352 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
9358 SmallVectorImpl<SDValue> &Results,
9360 if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
9364 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
9364 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
9370 SDValue Op = getNonRegisterValue(V);
9382 SDValue Chain = DAG.getEntryNode();
9511 FunctionLoweringInfo *FuncInfo, SmallVectorImpl<SDValue> &Chains,
9515 SDValue ArgVal, bool &ArgHasUses) {
9728 SmallVector<SDValue, 8> InVals;
9729 SDValue NewRoot = TLI->LowerFormalArguments(
9762 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
9778 SmallVector<SDValue, 4> Chains;
9781 SmallVector<SDValue, 4> ArgValues;
9846 SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
10034 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
10074 SDValue CondLHS = getValue(Cond);
10078 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
10080 SDValue Cond = DAG.getSetCC(
10097 SDValue BrCond =
lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h 100 DenseMap<const Value*, SDValue> NodeMap;
104 DenseMap<const Value*, SDValue> UnusedArgNodeMap;
133 SmallVector<SDValue, 8> PendingLoads;
144 SmallVector<SDValue, 8> PendingExports;
451 SDValue getRoot();
456 SDValue getControlRoot();
474 SDValue getCopyFromRegs(const Value *V, Type *Ty);
484 void resolveDanglingDebugInfo(const Value *V, SDValue Val);
500 SDValue getValue(const Value *V);
510 SDValue getNonRegisterValue(const Value *V);
511 SDValue getValueImpl(const Value *V);
513 void setValue(const Value *V, SDValue NewN) {
514 SDValue &N = NodeMap[V];
519 void setUnusedArgValue(const Value *V, SDValue NewN) {
520 SDValue &N = UnusedArgNodeMap[V];
540 void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall,
545 SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I,
546 SDValue Op);
550 unsigned NumArgs, SDValue Callee,
553 std::pair<SDValue, SDValue>
553 std::pair<SDValue, SDValue>
606 SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI);
613 void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee,
619 void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee,
773 SDValue Value, bool IsSigned);
784 bool IsDbgDeclare, const SDValue &N);
791 void updateDAGForMaybeTailCall(SDValue MaybeTC);
794 SDDbgValue *getDbgValue(SDValue N, DILocalVariable *Variable,
861 SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
862 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
862 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
870 void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl,
871 SDValue &Chain, SDValue *Flag, const Value *V = nullptr,
871 SDValue &Chain, SDValue *Flag, const Value *V = nullptr,
879 SelectionDAG &DAG, std::vector<SDValue> &Ops) const;
lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp 804 for (const SDValue &Op : N->op_values()) {
851 const SDValue Value) {
885 const SDValue Op = N->getOperand(i);
894 for (const SDValue &Op : N->op_values())
922 for (const SDValue &Op : N->op_values()) {
lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp 753 for (const SDValue &Op : N->op_values())
766 SDValue Src = N->getOperand(2);
1137 for (const SDValue &Op : N->op_values()) {
1971 bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
2000 bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
2030 void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
2032 std::vector<SDValue> InOps;
2067 std::vector<SDValue> SelOps;
2114 for (const SDValue &Op : ImmedUse->op_values()) {
2127 for (const SDValue &Op : Root->op_values()) {
2143 bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
2151 bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
2221 std::vector<SDValue> Ops(N->op_begin(), N->op_end());
2225 SDValue New = CurDAG->getNode(Branch ? ISD::INLINEASM_BR : ISD::INLINEASM, DL, VTs, Ops);
2238 SDValue New = CurDAG->getCopyFromReg(
2252 SDValue New = CurDAG->getCopyToReg(
2283 SDNode *NodeToMatch, SDValue InputChain,
2309 SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);
2340 static SDValue
2346 SmallVector<SDValue, 3> InputChains;
2355 std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2363 for (const SDValue &Op : V->op_values())
2385 for (SDValue V : InputChains)
2402 ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {
2463 SDValue N,
2464 const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2474 SDValue N,
2475 const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes,
2506 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2517 SDValue N, const TargetLowering *TLI, const DataLayout &DL,
2527 SDValue N) {
2534 SDValue N) {
2542 SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
2553 SDValue N) {
2564 SDValue N, unsigned ChildNo) {
2572 SDValue N, const SelectionDAGISel &SDISel) {
2585 SDValue N, const SelectionDAGISel &SDISel) {
2603 unsigned Index, SDValue N,
2606 SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
2689 SmallVector<SDValue, 4> NodeStack;
2698 SDValue InputChain, InputGlue;
2711 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes;
2716 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
2740 for (auto &J : I.NodeStack)
2804 SmallVector<SDValue, 8> NodeStack;
2805 SDValue N = SDValue(NodeToMatch, 0);
2815 SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;
2824 SDValue InputChain, InputGlue;
3025 SmallVector<SDValue, 8> Operands;
3235 SDValue Imm = RecordedNodes[RecNo].first;
3347 SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);
3403 SmallVector<SDValue, 8> Ops;
3424 SDValue V = NodeToMatch->getOperand(i);
3534 SDValue Res = RecordedNodes[ResSlot].first;
lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp 97 SDValue Op = EI.getNode()->getOperand(EI.getOperand());
lib/CodeGen/SelectionDAG/StatepointLowering.cpp 64 static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
92 SDValue
125 SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
225 SDValue Incoming = Builder.getValue(IncomingValue);
233 SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
266 SDValue Loc =
281 DenseMap<SDValue, const Value *> Seen;
286 SDValue SD = Builder.getValue(Ptrs[i]);
313 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
315 SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
316 SDValue ReturnValue, CallEndVal;
367 static std::tuple<SDValue, SDValue, MachineMemOperand*>
367 static std::tuple<SDValue, SDValue, MachineMemOperand*>
368 spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
368 spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
370 SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
414 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
415 SmallVectorImpl<SDValue> &Ops,
421 SDValue Chain = Builder.getRoot();
477 lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
551 SDValue Incoming;
586 SDValue Incoming = Builder.getValue(V);
608 SDValue SDV = Builder.getValue(V);
609 SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
634 SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
662 SmallVector<SDValue, 10> LoweredMetaArgs;
671 SDValue ReturnVal;
680 SDValue Chain = CallNode->getOperand(0);
682 SDValue Glue;
700 SmallVector<SDValue, 8> TSOps;
718 SDValue GCTransitionStart =
728 SmallVector<SDValue, 40> Ops;
742 SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
791 SmallVector<SDValue, 8> TEOps;
808 SDValue GCTransitionStart =
848 SDValue ActualCallee;
888 SDValue ReturnValue = LowerAsSTATEPOINT(SI);
907 SDValue Chain = DAG.getEntryNode();
926 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
952 if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
959 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
981 SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
1005 SDValue SD = getValue(DerivedPtr);
1020 SDValue SpillSlot = DAG.getTargetFrameIndex(Index, getFrameIndexTy());
1025 SDValue Chain = getRoot();
1038 SDValue SpillLoad = DAG.getLoad(LoadVT, getCurSDLoc(), Chain,
1049 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
lib/CodeGen/SelectionDAG/StatepointLowering.h 52 SDValue getLocation(SDValue Val) {
52 SDValue getLocation(SDValue Val) {
59 void setLocation(SDValue Val, SDValue Location) {
59 void setLocation(SDValue Val, SDValue Location) {
92 SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder);
111 DenseMap<SDValue, SDValue> Locations;
111 DenseMap<SDValue, SDValue> Locations;
lib/CodeGen/SelectionDAG/TargetLowering.cpp 52 SDValue &Chain) const {
77 const SmallVectorImpl<SDValue> &OutVals) const {
89 SDValue Value = OutVals[I];
121 std::pair<SDValue, SDValue>
121 std::pair<SDValue, SDValue>
123 ArrayRef<SDValue> Ops,
131 SDValue NewOp = Ops[i];
147 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
276 SDValue &NewLHS, SDValue &NewRHS,
276 SDValue &NewLHS, SDValue &NewRHS,
278 const SDLoc &dl, const SDValue OldLHS,
279 const SDValue OldRHS) const {
380 SDValue Ops[2] = {NewLHS, NewRHS};
393 SDValue Tmp = DAG.getNode(
422 SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
422 SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
468 bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
495 SDValue NewC = TLO.DAG.getConstant(Demanded & C, DL, VT);
496 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
510 bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
542 SDValue X = DAG.getNode(
547 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
554 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
569 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
584 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
585 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
603 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
610 if (SDValue V = SimplifyMultipleUseDemandedBits(
632 if (SDValue V = SimplifyMultipleUseDemandedBits(
651 if (SDValue V = SimplifyMultipleUseDemandedBits(
705 SDValue Vec = Op.getOperand(0);
738 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
754 SDValue Op, const APInt &OriginalDemandedBits,
815 SDValue Src = Op.getOperand(0);
837 SDValue Vec = Op.getOperand(0);
838 SDValue Scl = Op.getOperand(1);
875 SDValue Base = Op.getOperand(0);
876 SDValue Sub = Op.getOperand(1);
915 SDValue Src = Op.getOperand(0);
973 SDValue Op0 = Op.getOperand(0);
974 SDValue Op1 = Op.getOperand(1);
994 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
996 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1001 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1008 SDValue Op0 = Op.getOperand(0);
1009 SDValue Op1 = Op.getOperand(1);
1035 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1051 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1053 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1058 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1086 SDValue Op0 = Op.getOperand(0);
1087 SDValue Op1 = Op.getOperand(1);
1100 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1102 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1107 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1132 SDValue Op0 = Op.getOperand(0);
1133 SDValue Op1 = Op.getOperand(1);
1146 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1148 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1153 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1186 SDValue ANDC =
1197 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1246 SDValue Op0 = Op.getOperand(0);
1247 SDValue Op1 = Op.getOperand(1);
1274 SDValue Op0 = Op.getOperand(0);
1275 SDValue Op1 = Op.getOperand(1);
1303 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
1324 SDValue InnerOp = Op0.getOperand(0);
1332 SDValue NarrowShl =
1352 SDValue NewSA = TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
1354 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1371 SDValue Op0 = Op.getOperand(0);
1372 SDValue Op1 = Op.getOperand(1);
1409 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1430 SDValue Op0 = Op.getOperand(0);
1431 SDValue Op1 = Op.getOperand(1);
1481 SDValue NewSA =
1494 SDValue Op0 = Op.getOperand(0);
1495 SDValue Op1 = Op.getOperand(1);
1496 SDValue Op2 = Op.getOperand(2);
1532 SDValue Src = Op.getOperand(0);
1542 SDValue Op0 = Op.getOperand(0);
1559 SDValue ShiftAmt =
1622 SDValue Src = Op.getOperand(0);
1655 SDValue Src = Op.getOperand(0);
1703 SDValue Src = Op.getOperand(0);
1727 SDValue Src = Op.getOperand(0);
1738 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1760 SDValue Shift = Src.getOperand(1);
1774 SDValue NewTrunc =
1800 SDValue Src = Op.getOperand(0);
1801 SDValue Idx = Op.getOperand(1);
1827 SDValue Src = Op.getOperand(0);
1844 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
1849 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1923 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
1938 SDValue NewOp =
1947 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1949 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1956 SDValue NewOp =
1971 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
1976 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2016 bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2037 static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2057 SDValue Elt = BV->getOperand(Index);
2074 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2075 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2084 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2129 SDValue Src = Op.getOperand(0);
2219 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2233 SDValue SrcOp = Op.getOperand(i);
2248 SDValue SubOp = Op.getOperand(i);
2262 SDValue Base = Op.getOperand(0);
2263 SDValue Sub = Op.getOperand(1);
2294 SDValue Src = Op.getOperand(0);
2311 SDValue Vec = Op.getOperand(0);
2312 SDValue Scl = Op.getOperand(1);
2420 SDValue LegalShuffle =
2450 SDValue Src = Op.getOperand(0);
2577 void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2598 void TargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
2613 unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2627 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2639 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2651 SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
2652 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2664 SDValue
2665 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
2666 SDValue N1, MutableArrayRef<int> Mask,
2685 bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
2779 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2779 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2779 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2793 SDValue X, Y;
2805 SDValue Zero = DAG.getConstant(0, DL, OpVT);
2829 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
2830 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
2847 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
2848 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
2848 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
2864 SDValue X = N0->getOperand(0);
2926 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
2927 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
2928 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
2929 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
2935 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
2936 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
2936 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
2945 SDValue X, C, Y;
2986 SDValue Mask = N0.getOperand(1);
2999 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3000 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3001 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3008 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3008 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3008 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3021 SDValue X = N0.getOperand(0);
3022 SDValue Y = N0.getOperand(1);
3041 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3042 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3050 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3050 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3050 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3058 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3101 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3107 SDValue CTPOP = N0;
3116 SDValue CTOp = CTPOP.getOperand(0);
3121 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3122 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3123 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3133 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3134 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3136 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3137 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3138 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3139 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3150 SDValue PreExt;
3192 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3197 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3209 SDValue TopSetCC = N0->getOperand(0);
3276 SDValue Ptr = Lod->getBasePtr();
3281 SDValue NewLoad = DAG.getLoad(
3337 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3339 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3360 SDValue ZextOp;
3411 SDValue Val;
3430 SDValue Op0 = N0;
3479 if (SDValue V =
3579 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3598 SDValue ConstMinusOne =
3656 SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
3659 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
3687 SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
3689 SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
3712 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
3846 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
3852 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
3855 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
3868 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
3871 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
3879 SDValue Temp;
3946 SDValue N1 = N->getOperand(0);
3947 SDValue N2 = N->getOperand(1);
3964 SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4028 SDValue TargetLowering::LowerAsmOutputForConstraint(
4029 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
4029 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
4036 void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4038 std::vector<SDValue> &Ops,
4475 SDValue Op, SelectionDAG *DAG) {
4494 std::vector<SDValue> ResultOps;
4525 SDValue Op,
4562 static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
4565 SDValue Op0 = N->getOperand(0);
4566 SDValue Op1 = N->getOperand(1);
4573 SmallVector<SDValue, 16> Shifts, Factors;
4598 SDValue Shift, Factor;
4607 SDValue Res = Op0;
4621 SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
4635 SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
4654 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
4686 SDValue N0 = N->getOperand(0);
4687 SDValue N1 = N->getOperand(1);
4693 SDValue MagicFactor, Factor, Shift, ShiftMask;
4708 SDValue Q;
4714 SDValue LoHi =
4732 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
4733 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
4744 SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
4760 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
4804 SDValue N0 = N->getOperand(0);
4805 SDValue N1 = N->getOperand(1);
4811 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
4823 SDValue Q = N0;
4834 SDValue LoHi =
4849 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
4868 SDValue One = DAG.getConstant(1, dl, VT);
4869 SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
4878 turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
4879 std::function<bool(SDValue)> Predicate,
4880 SDValue AlternativeReplacement = SDValue()) {
4881 SDValue Replacement;
4906 SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
4906 SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
4907 SDValue CompTargetNode,
4912 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
4922 SDValue
4923 TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
4924 SDValue CompTargetNode, ISD::CondCode Cond,
4955 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
5009 SDValue N = REMNode.getOperand(0);
5010 SDValue D = REMNode.getOperand(1);
5025 SDValue PVal, KVal, QVal;
5047 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5073 SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5073 SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5074 SDValue CompTargetNode,
5079 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5090 SDValue
5091 TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5092 SDValue CompTargetNode, ISD::CondCode Cond,
5130 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5212 SDValue N = REMNode.getOperand(0);
5213 SDValue D = REMNode.getOperand(1);
5228 SDValue PVal, AVal, KVal, QVal;
5256 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5283 SDValue Fold =
5304 SDValue IntMin = DAG.getConstant(
5306 SDValue IntMax = DAG.getConstant(
5308 SDValue Zero =
5312 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5316 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5318 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5325 SDValue Blended =
5332 verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5342 char TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
5458 SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
5458 SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
5477 SmallVector<SDValue, 4> Ops;
5478 for (SDValue C : Op->op_values()) {
5543 SDValue Neg2 = getNegatedExpression(Op.getOperand(2), DAG, LegalOperations,
5552 SDValue Neg0 = getNegatedExpression(
5559 SDValue Neg1 = getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
5587 SDValue LHS, SDValue RHS,
5587 SDValue LHS, SDValue RHS,
5588 SmallVectorImpl<SDValue> &Result,
5590 MulExpansionKind Kind, SDValue LL,
5591 SDValue LH, SDValue RL, SDValue RH) const {
5591 SDValue LH, SDValue RL, SDValue RH) const {
5591 SDValue LH, SDValue RL, SDValue RH) const {
5632 SDValue Lo, Hi;
5651 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
5678 SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
5714 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
5725 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
5737 SDValue Carry = Next.getValue(1);
5754 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
5769 bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
5769 bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
5771 SDValue LL, SDValue LH, SDValue RL,
5771 SDValue LL, SDValue LH, SDValue RL,
5771 SDValue LL, SDValue LH, SDValue RL,
5772 SDValue RH) const {
5773 SmallVector<SDValue, 2> Result;
5785 bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
5797 SDValue X = Node->getOperand(0);
5798 SDValue Y = Node->getOperand(1);
5799 SDValue Z = Node->getOperand(2);
5806 SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
5807 SDValue Zero = DAG.getConstant(0, DL, ShVT);
5809 SDValue ShAmt;
5811 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
5817 SDValue InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
5818 SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
5819 SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
5820 SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
5828 SDValue IsZeroShift = DAG.getSetCC(DL, CCVT, ShAmt, Zero, ISD::SETEQ);
5834 bool TargetLowering::expandROT(SDNode *Node, SDValue &Result,
5839 SDValue Op0 = Node->getOperand(0);
5840 SDValue Op1 = Node->getOperand(1);
5844 SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
5849 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
5869 SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
5870 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
5871 SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
5872 SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
5878 bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
5881 SDValue Src = Node->getOperand(OpNo);
5904 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
5905 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
5906 SDValue Bias = DAG.getConstant(127, dl, IntVT);
5907 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
5908 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
5909 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
5911 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
5913 SDValue ExponentBits = DAG.getNode(
5916 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
5918 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
5923 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
5941 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
5949 bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
5950 SDValue &Chain,
5954 SDValue Src = Node->getOperand(OpNo);
5985 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
5986 SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
6000 SDValue SrcBiased;
6006 SDValue Val = DAG.getSelect(dl, SrcVT, Sel, Src, SrcBiased);
6007 SDValue Ofs = DAG.getSelect(dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT),
6009 SDValue SInt;
6023 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6025 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
6034 bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
6036 SDValue Src = Node->getOperand(0);
6059 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
6061 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
6062 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Src, ShiftConst);
6063 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
6064 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Src, AndConst);
6065 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
6067 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Or);
6068 SDValue Slow = DAG.getNode(ISD::FADD, dl, DstVT, SignCvt, SignCvt);
6077 SDValue SignBitTest = DAG.getSetCC(
6098 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
6099 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
6101 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
6102 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
6103 SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
6105 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
6106 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
6107 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
6108 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
6109 SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
6110 SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
6111 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
6119 SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
6126 SDValue Quiet0 = Node->getOperand(0);
6127 SDValue Quiet1 = Node->getOperand(1);
6159 bool TargetLowering::expandCTPOP(SDNode *Node, SDValue &Result,
6164 SDValue Op = Node->getOperand(0);
6182 SDValue Mask55 =
6184 SDValue Mask33 =
6186 SDValue Mask0F =
6188 SDValue Mask01 =
6219 bool TargetLowering::expandCTLZ(SDNode *Node, SDValue &Result,
6224 SDValue Op = Node->getOperand(0);
6238 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
6239 SDValue Zero = DAG.getConstant(0, dl, VT);
6240 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
6263 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
6272 bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
6276 SDValue Op = Node->getOperand(0);
6290 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
6291 SDValue Zero = DAG.getConstant(0, dl, VT);
6292 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
6311 SDValue Tmp = DAG.getNode(
6327 bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
6332 SDValue Op = N->getOperand(0);
6340 SDValue Shift =
6343 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
6348 SDValue TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
6351 SDValue Chain = LD->getChain();
6352 SDValue BasePTR = LD->getBasePtr();
6364 SmallVector<SDValue, 8> Vals;
6365 SmallVector<SDValue, 8> LoadChains;
6368 SDValue ScalarLoad =
6380 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
6381 SDValue Value = DAG.getBuildVector(LD->getValueType(0), SL, Vals);
6386 SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
6390 SDValue Chain = ST->getChain();
6391 SDValue BasePtr = ST->getBasePtr();
6392 SDValue Value = ST->getValue();
6415 SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
6418 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
6420 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
6421 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
6424 SDValue ShiftAmount =
6426 SDValue ShiftedElt =
6441 SmallVector<SDValue, 8> Stores;
6443 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
6446 SDValue Ptr = DAG.getObjectPtrOffset(SL, BasePtr, Idx * Stride);
6449 SDValue Store = DAG.getTruncStore(
6460 std::pair<SDValue, SDValue>
6460 std::pair<SDValue, SDValue>
6464 SDValue Chain = LD->getChain();
6465 SDValue Ptr = LD->getBasePtr();
6477 SDValue Scalarized = scalarizeVectorLoad(LD, DAG);
6485 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
6487 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
6503 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
6505 SmallVector<SDValue, 8> Stores;
6506 SDValue StackPtr = StackBase;
6512 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
6513 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
6518 SDValue Load = DAG.getLoad(
6536 SDValue Load =
6549 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
6579 SDValue Lo, Hi;
6603 SDValue ShiftAmount =
6606 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
6609 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
6615 SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
6619 SDValue Chain = ST->getChain();
6620 SDValue Ptr = ST->getBasePtr();
6621 SDValue Val = ST->getValue();
6634 SDValue Result = scalarizeVectorStore(ST, DAG);
6640 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
6656 SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
6660 SDValue Store = DAG.getTruncStore(
6666 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
6667 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
6668 SmallVector<SDValue, 8> Stores;
6674 SDValue Load = DAG.getLoad(
6695 SDValue Load = DAG.getExtLoad(
6705 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
6717 SDValue ShiftAmount = DAG.getConstant(
6719 SDValue Lo = Val;
6720 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
6723 SDValue Store1, Store2;
6736 SDValue Result =
6741 SDValue
6742 TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
6742 TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
6746 SDValue Increment;
6754 SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
6764 SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
6773 static SDValue clampDynamicVectorIndex(SelectionDAG &DAG,
6774 SDValue Idx,
6793 SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
6794 SDValue VecPtr, EVT VecVT,
6795 SDValue Index) const {
6820 SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
6839 SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
6844 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
6844 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
6857 SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
6857 SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
6867 SDValue Zext = Op.getOperand(0);
6873 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
6874 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
6882 SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
6884 SDValue LHS = Node->getOperand(0);
6885 SDValue RHS = Node->getOperand(1);
6894 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
6899 SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
6900 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
6925 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT),
6927 SDValue SumDiff = Result.getValue(0);
6928 SDValue Overflow = Result.getValue(1);
6929 SDValue Zero = DAG.getConstant(0, dl, VT);
6930 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
6935 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
6943 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
6944 SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
6954 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
6955 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
6956 SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT);
6962 SDValue
6971 SDValue LHS = Node->getOperand(0);
6972 SDValue RHS = Node->getOperand(1);
6988 SDValue Result =
6990 SDValue Product = Result.getValue(0);
6991 SDValue Overflow = Result.getValue(1);
6992 SDValue Zero = DAG.getConstant(0, dl, VT);
6996 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
6997 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
6998 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
7002 SDValue Result =
7004 SDValue Product = Result.getValue(0);
7005 SDValue Overflow = Result.getValue(1);
7008 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
7020 SDValue Lo, Hi;
7024 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
7046 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
7058 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
7070 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
7071 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
7074 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
7076 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
7079 SDValue Zero = DAG.getConstant(0, dl, VT);
7080 SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
7090 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
7095 SDValue HighMask =
7103 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
7103 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
7105 SDValue LHS = Node->getOperand(0);
7106 SDValue RHS = Node->getOperand(1);
7112 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
7113 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
7127 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
7132 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
7132 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
7134 SDValue LHS = Node->getOperand(0);
7135 SDValue RHS = Node->getOperand(1);
7148 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
7149 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
7154 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
7162 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
7163 SDValue ConditionRHS =
7171 bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
7172 SDValue &Overflow, SelectionDAG &DAG) const {
7176 SDValue LHS = Node->getOperand(0);
7177 SDValue RHS = Node->getOperand(1);
7188 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);
7203 SDValue BottomHalf;
7204 SDValue TopHalf;
7218 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
7220 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,
7243 SDValue HiLHS;
7244 SDValue HiRHS;
7266 SDValue Ret;
7275 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
7278 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
7295 SDValue ShiftAmt = DAG.getConstant(
7298 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
7315 SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
7340 SDValue Op = Node->getOperand(0);
7350 SDValue Lo, Hi;
7360 SmallVector<SDValue, 8> Ops;
7363 SDValue Res = Ops[0];
lib/Target/AArch64/AArch64ISelDAGToDAG.cpp 64 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
66 std::vector<SDValue> &OutOps) override;
70 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
70 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
70 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
71 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
71 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
71 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
72 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
72 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
72 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
73 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
73 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
73 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
76 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
76 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
76 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
79 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
79 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
79 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
82 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
82 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
82 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
85 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
85 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
85 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
88 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
88 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
88 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
91 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
91 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
91 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
94 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
94 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
94 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
97 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
97 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
97 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
100 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
100 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
100 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
103 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
103 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
103 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
106 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
106 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
106 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
109 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
109 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
109 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
112 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
112 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
112 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
115 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
115 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
115 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
118 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
118 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
118 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
121 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
121 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
121 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
124 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
124 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
124 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
127 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
127 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
127 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
132 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
132 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
132 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
133 SDValue &SignExtend, SDValue &DoShift) {
133 SDValue &SignExtend, SDValue &DoShift) {
138 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
138 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
138 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
139 SDValue &SignExtend, SDValue &DoShift) {
139 SDValue &SignExtend, SDValue &DoShift) {
143 bool SelectDupZeroOrUndef(SDValue N) {
149 auto Opnd0 = N->getOperand(0);
169 SDValue createDTuple(ArrayRef<SDValue> Vecs);
169 SDValue createDTuple(ArrayRef<SDValue> Vecs);
170 SDValue createQTuple(ArrayRef<SDValue> Vecs);
170 SDValue createQTuple(ArrayRef<SDValue> Vecs);
174 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
174 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
209 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
209 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
210 SDValue &Shift);
211 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
211 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
212 SDValue &OffImm) {
215 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
216 unsigned Size, SDValue &Base,
217 SDValue &OffImm);
218 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
218 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
219 SDValue &OffImm);
220 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
220 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
221 SDValue &OffImm);
222 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
222 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
223 SDValue &Offset, SDValue &SignExtend,
223 SDValue &Offset, SDValue &SignExtend,
224 SDValue &DoShift);
225 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
225 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
226 SDValue &Offset, SDValue &SignExtend,
226 SDValue &Offset, SDValue &SignExtend,
227 SDValue &DoShift);
228 bool isWorthFolding(SDValue V) const;
229 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
230 SDValue &Offset, SDValue &SignExtend);
230 SDValue &Offset, SDValue &SignExtend);
233 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
233 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
237 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
237 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
256 static bool isIntImmediate(SDValue N, uint64_t &Imm) {
270 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
270 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
282 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64);
283 SDValue NewOp =
296 bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
296 bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
297 SDValue &Shift) {
326 bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
326 bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
327 SDValue &Shift) {
359 static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
376 static bool isWorthFoldingSHL(SDValue V) {
399 bool AArch64DAGToDAGISel::isWorthFolding(SDValue V) const {
410 const SDValue LHS = V.getOperand(0);
411 const SDValue RHS = V.getOperand(1);
427 bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
428 SDValue &Reg, SDValue &Shift) {
428 SDValue &Reg, SDValue &Shift) {
451 getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
503 static bool checkHighLaneIndex(SDNode *DL, SDValue &LaneOp, int &LaneIdx) {
508 SDValue SV = DL->getOperand(0);
512 SDValue EV = SV.getOperand(1);
526 static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp,
526 static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp,
526 static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp,
527 SDValue &LaneOp, int &LaneIdx) {
543 SDValue Op0 = N->getOperand(0);
544 SDValue Op1 = N->getOperand(1);
545 SDValue MLAOp1; // Will hold ordinary multiplicand for MLA.
546 SDValue MLAOp2; // Will hold lane-accessed multiplicand for MLA.
559 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
561 SDValue Ops[] = { Op0, MLAOp1, MLAOp2, LaneIdxVal };
588 SDValue SMULLOp0;
589 SDValue SMULLOp1;
596 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
598 SDValue Ops[] = { SMULLOp0, SMULLOp1, LaneIdxVal };
635 static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
635 static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
640 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
649 bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
649 bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
650 SDValue &Shift) {
697 static bool isWorthFoldingADDlow(SDValue N) {
716 bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
718 SDValue &Base,
719 SDValue &OffImm) {
779 bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
780 SDValue &Base, SDValue &OffImm) {
780 SDValue &Base, SDValue &OffImm) {
846 bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
847 SDValue &Base,
848 SDValue &OffImm) {
872 static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
872 static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
874 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
875 SDValue ImpDef = SDValue(
884 bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
885 bool WantExtend, SDValue &Offset,
886 SDValue &SignExtend) {
916 bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
917 SDValue &Base, SDValue &Offset,
917 SDValue &Base, SDValue &Offset,
918 SDValue &SignExtend,
919 SDValue &DoShift) {
922 SDValue LHS = N.getOperand(0);
923 SDValue RHS = N.getOperand(1);
1005 bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1006 SDValue &Base, SDValue &Offset,
1006 SDValue &Base, SDValue &Offset,
1007 SDValue &SignExtend,
1008 SDValue &DoShift) {
1011 SDValue LHS = N.getOperand(0);
1012 SDValue RHS = N.getOperand(1);
1045 SDValue Ops[] = { RHS };
1048 SDValue MOVIV = SDValue(MOVI, 0);
1081 SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1081 SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1090 SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1090 SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1099 SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1099 SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1111 SmallVector<SDValue, 4> Ops;
1137 SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Off,
1139 SDValue RegSeq = createQTuple(Regs);
1141 SmallVector<SDValue, 6> Ops;
1215 SDValue Chain = LD->getChain();
1216 SDValue Base = LD->getBasePtr();
1220 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1221 SDValue Ops[] = { Base, Offset, Chain };
1225 SDValue LoadedVal = SDValue(Res, 1);
1227 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1247 SDValue Chain = N->getOperand(0);
1249 SDValue Ops[] = {N->getOperand(2), // Mem operand;
1255 SDValue SuperReg = SDValue(Ld, 0);
1273 SDValue Chain = N->getOperand(0);
1275 SDValue Ops[] = {N->getOperand(1), // Mem operand
1288 SDValue SuperReg = SDValue(Ld, 1);
1308 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1309 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1311 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)};
1330 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1331 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1333 SDValue Ops[] = {RegSeq,
1351 SDValue operator()(SDValue V64Reg) {
1351 SDValue operator()(SDValue V64Reg) {
1358 SDValue Undef =
1367 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
1367 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
1384 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1390 SDValue RegSeq = createQTuple(Regs);
1397 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1400 SDValue SuperReg = SDValue(Ld, 0);
1406 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
1423 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1429 SDValue RegSeq = createQTuple(Regs);
1437 SDValue Ops[] = {RegSeq,
1449 SDValue SuperReg = SDValue(Ld, 1);
1458 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
1478 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1484 SDValue RegSeq = createQTuple(Regs);
1489 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1507 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1513 SDValue RegSeq = createQTuple(Regs);
1521 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1535 unsigned &Opc, SDValue &Opd0,
1627 SDValue &Opd0, unsigned &Immr,
1636 SDValue Op = N->getOperand(0);
1660 SDValue &Opd0, unsigned &LSB,
1705 static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
1781 SDValue Op = N->getOperand(0);
1787 SDValue Opd0 = Widen(CurDAG, Op.getOperand(0));
1790 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1797 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
1839 SDValue Opd0;
1849 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
1853 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1859 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1897 static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
1899 static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
1908 static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
1935 static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
1945 static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
1973 static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
1973 static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
2029 SDValue Orig, unsigned Depth) {
2076 static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
2103 static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
2103 static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
2133 static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
2135 SDValue &Src, int &ShiftAmount,
2213 SDValue And = N->getOperand(0);
2273 SDValue Ops[] = {And.getOperand(0), SDValue(MOVI, 0),
2317 SDValue Dst, Src;
2320 SDValue OrOpd0Val = N->getOperand(I % 2);
2322 SDValue OrOpd1Val = N->getOperand((I + 1) % 2);
2385 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(ImmR, DL, VT),
2396 SDValue And0 = N->getOperand(0);
2397 SDValue And1 = N->getOperand(1);
2412 SDValue Src = And1->getOperand(0);
2413 SDValue Dst = And0->getOperand(0);
2430 SDValue Ops[] = {Dst, SDValue(LSR, 0),
2471 SDValue Op0;
2483 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT),
2524 SDValue ShiftAmt = N->getOperand(1);
2526 SDValue NewShiftAmt;
2534 SDValue Add0 = ShiftAmt->getOperand(0);
2535 SDValue Add1 = ShiftAmt->getOperand(1);
2558 SDValue Zero =
2583 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32);
2590 SDValue Ops[] = {N->getOperand(0), NewShiftAmt};
2596 AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
2596 AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
2801 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2825 SDValue IRG_SP = N->getOperand(2);
2835 SDValue FiOp = CurDAG->getTargetFrameIndex(
2943 SDValue New = CurDAG->getCopyFromReg(
2948 SDValue New = CurDAG->getCopyFromReg(
2962 SDValue TFI = CurDAG->getTargetFrameIndex(
2965 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
2979 SDValue MemAddr = Node->getOperand(2);
2981 SDValue Chain = Node->getOperand(0);
2998 SDValue Chain = Node->getOperand(0);
2999 SDValue ValLo = Node->getOperand(2);
3000 SDValue ValHi = Node->getOperand(3);
3001 SDValue MemAddr = Node->getOperand(4);
3004 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
lib/Target/AArch64/AArch64ISelLowering.cpp 913 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
989 SDValue New;
1000 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1009 SDValue Op, const APInt &Demanded, TargetLoweringOpt &TLO) const {
1053 const SDValue Op, KnownBits &Known,
1597 static bool isCMN(SDValue Op, ISD::CondCode CC) {
1602 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1602 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1602 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1700 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
1700 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
1700 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
1701 ISD::CondCode CC, SDValue CCOp,
1717 SDValue SubOp0 = RHS.getOperand(0);
1727 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
1730 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
1748 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
1766 SDValue O0 = Val->getOperand(0);
1767 SDValue O1 = Val->getOperand(1);
1810 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
1810 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
1811 AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
1816 SDValue LHS = Val->getOperand(0);
1817 SDValue RHS = Val->getOperand(1);
1833 SDValue ExtraCmp;
1855 SDValue LHS = Val->getOperand(0);
1862 SDValue RHS = Val->getOperand(1);
1909 SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
1912 SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
1922 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
1922 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
1936 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
1970 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1970 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1970 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1971 SDValue &AArch64cc, SelectionDAG &DAG,
2040 SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2048 SDValue Cmp;
2075 SDValue SExt =
2101 static std::pair<SDValue, SDValue>
2101 static std::pair<SDValue, SDValue>
2102 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2105 SDValue Value, Overflow;
2107 SDValue LHS = Op.getOperand(0);
2108 SDValue RHS = Op.getOperand(1);
2142 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2143 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2156 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2159 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2172 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2186 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2187 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2195 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2216 SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
2216 SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
2218 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2225 static bool isOverflowIntrOpRes(SDValue Op) {
2232 static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
2232 static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
2233 SDValue Sel = Op.getOperand(0);
2234 SDValue Other = Op.getOperand(1);
2249 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2250 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2252 SDValue Value, Overflow;
2254 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2272 SDValue LHS = Sel.getOperand(0);
2273 SDValue RHS = Sel.getOperand(1);
2274 SDValue TVal = Sel.getOperand(2);
2275 SDValue FVal = Sel.getOperand(3);
2298 SDValue CCVal;
2299 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
2312 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2312 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2348 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
2348 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
2356 SDValue Value, Overflow;
2360 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2361 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2366 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2379 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
2379 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
2405 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
2405 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
2415 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
2415 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
2428 SDValue SrcVal = Op.getOperand(0);
2434 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
2434 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
2455 SDValue Cv =
2466 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
2474 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
2474 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
2499 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2504 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
2504 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
2510 SDValue In = Op.getOperand(0);
2532 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2532 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2565 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2565 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2570 SDValue Arg = Op.getOperand(0);
2586 SDValue Callee =
2595 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2595 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2599 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2599 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2631 static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
2631 static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
2655 for (const SDValue &Elt : N->op_values()) {
2674 static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
2687 SmallVector<SDValue, 8> Ops;
2730 SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
2730 SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
2738 SDValue FPCR_64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i64,
2741 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
2742 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
2744 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
2750 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
2750 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
2797 SDValue Op0;
2798 SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
2809 SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
2810 SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
2819 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2819 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2832 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
2880 SDValue FnOp = Op.getOperand(1);
2881 SDValue IncomingFPOp = Op.getOperand(2);
2892 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
2897 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
2903 SDValue Value = ST->getValue();
2912 SDValue Undef = DAG.getUNDEF(MVT::i16);
2913 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
2916 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
2918 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
2921 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
2931 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
2931 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
2937 SDValue Value = StoreNode->getValue();
2959 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
2959 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
3134 SDValue AArch64TargetLowering::LowerFormalArguments(
3135 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3137 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3144 DenseMap<unsigned, SDValue> CopiedRegs;
3180 SmallVector<SDValue, 16> ArgValues;
3195 SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
3201 SDValue ArgValue;
3269 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3358 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
3394 SDValue &Chain) const {
3401 SmallVector<SDValue, 8> MemOps;
3420 SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
3424 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
3425 SDValue Store = DAG.getStore(
3452 SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
3456 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
3458 SDValue Store = DAG.getStore(
3477 SDValue AArch64TargetLowering::LowerCallResult(
3478 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
3478 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
3480 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
3481 SDValue ThisVal) const {
3487 DenseMap<unsigned, SDValue> CopiedRegs;
3507 SDValue Val = CopiedRegs.lookup(VA.getLocReg());
3559 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
3561 const SmallVectorImpl<SDValue> &OutVals,
3675 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
3675 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
3679 SmallVector<SDValue, 8> ArgChains;
3715 SDValue
3717 SmallVectorImpl<SDValue> &InVals) const {
3721 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
3723 SDValue Chain = CLI.Chain;
3724 SDValue Callee = CLI.Callee;
3843 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
3846 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3848 SmallVector<SDValue, 8> MemOpChains;
3854 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
3863 SDValue Arg = OutVals[realArgIdx];
3921 SDValue &Bits =
3946 SDValue DstAddr;
3962 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3978 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3986 SDValue SizeNode =
3988 SDValue Cpy = DAG.getMemcpy(
4003 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
4014 SDValue InFlag;
4060 std::vector<SDValue> Ops;
4122 SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
4159 SDValue
4160 AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4163 const SmallVectorImpl<SDValue> &OutVals,
4177 SDValue Flag;
4178 SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
4184 SDValue Arg = OutVals[realRVLocIdx];
4214 SDValue &Bits =
4227 SmallVector<SDValue, 4> RetOps(1, Chain);
4240 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
4278 SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
4285 SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
4291 SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
4298 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
4306 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
4311 SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
4319 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
4335 SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4340 SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
4341 SDValue Lo = getTargetNode(N, Ty, DAG,
4343 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
4349 SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
4354 SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
4358 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
4358 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
4374 SDValue Result;
4418 SDValue
4419 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
4429 SDValue TLVPAddr =
4431 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
4435 SDValue Chain = DAG.getEntryNode();
4436 SDValue FuncTLVGet = DAG.getLoad(
4486 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
4486 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
4491 SDValue Chain = DAG.getEntryNode();
4496 SDValue Glue = Chain.getValue(1);
4501 SDValue
4502 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
4523 SDValue TPOff;
4528 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
4531 SDValue HiVar = DAG.getTargetGlobalAddress(
4533 SDValue LoVar = DAG.getTargetGlobalAddress(
4537 SDValue TPWithOff_lo =
4542 SDValue TPWithOff =
4565 SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
4574 SDValue HiVar = DAG.getTargetGlobalAddress(
4576 SDValue LoVar = DAG.getTargetGlobalAddress(
4590 SDValue SymAddr =
4601 SDValue
4602 AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
4606 SDValue Chain = DAG.getEntryNode();
4610 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
4614 SDValue TLSArray =
4623 SDValue TLSIndexHi =
4625 SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
4627 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
4628 SDValue TLSIndex =
4636 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
4638 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
4645 SDValue TGAHi = DAG.getTargetGlobalAddress(
4647 SDValue TGALo = DAG.getTargetGlobalAddress(
4652 SDValue Addr =
4660 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
4660 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
4676 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4676 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4677 SDValue Chain = Op.getOperand(0);
4679 SDValue LHS = Op.getOperand(2);
4680 SDValue RHS = Op.getOperand(3);
4681 SDValue Dest = Op.getOperand(4);
4715 SDValue Value, Overflow;
4720 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
4742 SDValue Test = LHS.getOperand(0);
4758 SDValue Test = LHS.getOperand(0);
4785 SDValue CCVal;
4786 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4796 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4799 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4800 SDValue BR1 =
4803 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4811 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
4811 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
4816 SDValue In1 = Op.getOperand(0);
4817 SDValue In2 = Op.getOperand(1);
4827 SDValue VecVal1, VecVal2;
4862 SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
4872 SDValue Sel =
4885 SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
4885 SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
4901 SDValue Val = Op.getOperand(0);
4910 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
4911 SDValue UaddLV = DAG.getNode(
4943 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
4943 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
4948 SDValue LHS = Op.getOperand(0);
4949 SDValue RHS = Op.getOperand(1);
4955 SDValue TVal = DAG.getConstant(1, dl, VT);
4956 SDValue FVal = DAG.getConstant(0, dl, VT);
4972 SDValue CCVal;
4973 SDValue Cmp =
4988 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4994 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5007 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5008 SDValue CS1 =
5011 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5016 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
5016 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
5017 SDValue RHS, SDValue TVal,
5017 SDValue RHS, SDValue TVal,
5018 SDValue FVal, const SDLoc &dl,
5153 SDValue CCVal;
5154 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
5164 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
5190 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5191 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
5196 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5204 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
5204 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
5207 SDValue LHS = Op.getOperand(0);
5208 SDValue RHS = Op.getOperand(1);
5209 SDValue TVal = Op.getOperand(2);
5210 SDValue FVal = Op.getOperand(3);
5215 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
5215 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
5217 SDValue CCVal = Op->getOperand(0);
5218 SDValue TVal = Op->getOperand(1);
5219 SDValue FVal = Op->getOperand(2);
5230 SDValue Value, Overflow;
5232 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
5240 SDValue LHS, RHS;
5253 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
5253 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
5268 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
5268 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
5273 SDValue JT = Op.getOperand(1);
5274 SDValue Entry = Op.getOperand(2);
5284 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
5284 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
5301 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
5301 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
5313 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
5313 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
5319 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
5327 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
5327 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
5333 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
5342 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
5342 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
5351 SDValue Chain = Op.getOperand(0);
5352 SDValue VAList = Op.getOperand(1);
5354 SmallVector<SDValue, 4> MemOps;
5357 SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
5364 SDValue GRTop, GRTopAddr;
5381 SDValue VRTop, VRTopAddr;
5395 SDValue GROffsAddr =
5402 SDValue VROffsAddr =
5411 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
5411 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
5423 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
5423 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
5440 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
5440 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
5447 SDValue Chain = Op.getOperand(0);
5448 SDValue Addr = Op.getOperand(1);
5453 SDValue VAList =
5482 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5487 SDValue APStore =
5493 SDValue WideFP =
5496 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
5498 SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
5506 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
5506 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
5514 SDValue FrameAddr =
5527 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
5527 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
5557 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
5557 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
5564 SDValue FrameAddr =
5566 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5571 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
5571 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
5581 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5582 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5595 SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
5595 SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
5601 SDValue ShOpLo = Op.getOperand(0);
5602 SDValue ShOpHi = Op.getOperand(1);
5603 SDValue ShAmt = Op.getOperand(2);
5608 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5610 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
5614 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5616 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5621 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5624 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
5625 SDValue LoForNormalShift =
5631 SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
5632 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5637 SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
5638 SDValue HiForBigShift =
5643 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5646 SDValue Ops[2] = { Lo, Hi };
5652 SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
5652 SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
5658 SDValue ShOpLo = Op.getOperand(0);
5659 SDValue ShOpHi = Op.getOperand(1);
5660 SDValue ShAmt = Op.getOperand(2);
5663 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5665 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
5669 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5671 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5676 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5678 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
5679 SDValue HiForNormalShift =
5682 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
5687 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5692 SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
5693 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
5694 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5697 SDValue Ops[2] = { Lo, Hi };
5749 static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
5750 SDValue Operand, SelectionDAG &DAG,
5771 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
5771 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
5778 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
5789 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
5797 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5798 SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ);
5813 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
5813 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
5817 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
5828 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
6064 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
6064 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
6066 SDValue Result;
6221 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
6221 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
6234 static unsigned getExtFactor(SDValue &V) {
6241 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
6241 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
6253 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
6253 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
6262 SDValue Vec;
6269 SDValue ShuffleVec;
6276 ShuffleSourceInfo(SDValue Vec)
6280 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
6287 SDValue V = Op.getOperand(i);
6301 SDValue SourceVec = V.getOperand(0);
6379 SDValue VEXTSrc1 =
6382 SDValue VEXTSrc2 =
6416 SDValue Entry = Op.getOperand(i);
6447 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
6451 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
6453 SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
6717 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
6717 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
6720 SDValue V0 = Op.getOperand(0);
6721 SDValue V1 = Op.getOperand(1);
6747 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
6747 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
6748 SDValue RHS, SelectionDAG &DAG,
6779 SDValue OpLHS, OpRHS;
6818 SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
6849 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
6849 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
6852 SDValue V1 = Op.getOperand(0);
6853 SDValue V2 = Op.getOperand(1);
6859 SmallVector<SDValue, 8> TBLMask;
6874 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
6875 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
6877 SDValue Shuffle;
6924 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
6924 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
6937 SDValue V1 = Op.getOperand(0);
6938 SDValue V2 = Op.getOperand(1);
7022 if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
7029 SDValue DstVec = DstIsLeft ? V1 : V2;
7030 SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
7032 SDValue SrcVec = V1;
7038 SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
7076 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
7076 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
7082 SDValue SplatVal = Op.getOperand(0);
7134 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7134 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7145 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
7155 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7155 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7157 const SDValue *LHS = nullptr) {
7184 SDValue Mov;
7203 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7203 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7205 const SDValue *LHS = nullptr) {
7224 SDValue Mov;
7243 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
7243 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
7263 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
7274 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7274 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7285 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
7295 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7295 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7316 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
7328 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
7363 static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
7372 const SDValue And = N->getOperand(0);
7377 SDValue Shift = N->getOperand(1);
7405 SDValue X = And.getOperand(0);
7406 SDValue Y = Shift.getOperand(0);
7410 SDValue ResultSLI =
7424 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
7424 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
7428 if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
7434 SDValue LHS = Op.getOperand(0);
7448 SDValue NewOp;
7469 static SDValue NormalizeBuildVector(SDValue Op,
7469 static SDValue NormalizeBuildVector(SDValue Op,
7479 SmallVector<SDValue, 16> Ops;
7480 for (SDValue Lane : Op->ops()) {
7500 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG) {
7500 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG) {
7507 SDValue NewOp;
7541 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
7541 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
7563 if (SDValue V = ConstantBuildVector(Op, DAG))
7586 SDValue Value;
7587 SDValue ConstantValue;
7589 SDValue V = Op.getOperand(i);
7635 SDValue V = Op.getOperand(i);
7639 SDValue N0 = N->getOperand(0);
7673 SDValue LHS =
7676 SDValue RHS =
7702 SDValue Lane = Value.getOperand(1);
7716 SmallVector<SDValue, 8> Ops;
7727 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
7742 SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue),
7752 SDValue V = Op.getOperand(i);
7753 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
7772 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7787 SDValue Vec = DAG.getUNDEF(VT);
7788 SDValue Op0 = Op.getOperand(0);
7809 SDValue V = Op.getOperand(i);
7812 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
7824 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
7824 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
7848 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
7851 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec,
7857 SDValue
7858 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
7882 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
7894 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
7894 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
7962 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
7982 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
7993 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
8001 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
8001 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
8039 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
8040 SDValue NegShiftLeft =
8050 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
8050 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
8050 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
8068 SDValue Fcmeq;
8107 SDValue Cmeq;
8145 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
8145 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
8148 SDValue LHS = Op.getOperand(0);
8149 SDValue RHS = Op.getOperand(1);
8156 SDValue Cmp =
8170 SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
8187 SDValue Cmp =
8193 SDValue Cmp2 =
8209 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
8209 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
8211 SDValue VecOp = ScalarOp.getOperand(0);
8212 auto Rdx = DAG.getNode(Op, DL, VecOp.getSimpleValueType(), VecOp);
8217 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
8217 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
8250 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
8250 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
8259 SDValue RHS = Op.getOperand(2);
8267 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
8267 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
8276 SDValue RHS = Op.getOperand(2);
8284 SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
8285 SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
8285 SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
8285 SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
8288 SDValue Callee = DAG.getTargetExternalSymbol("__chkstk", PtrVT, 0);
8312 SDValue
8313 AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
8320 SDValue Chain = Op.getOperand(0);
8321 SDValue Size = Op.getOperand(1);
8327 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
8334 SDValue Ops[2] = {SP, Chain};
8342 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
8353 SDValue Ops[2] = {SP, Chain};
8477 const SDValue &Base = Mem->getBasePtr();
8553 bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
9269 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
9277 SDValue Shift = N->getOperand(0);
9278 SDValue Ones = N->getOperand(1);
9293 static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
9296 SDValue N0 = N->getOperand(0);
9297 SDValue N1 = N->getOperand(1);
9307 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
9310 SDValue Cmp =
9320 static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
9326 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
9332 SDValue
9347 SDValue N0 = N->getOperand(0);
9349 SDValue Zero = DAG.getConstant(0, DL, VT);
9350 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
9353 SDValue CCVal;
9354 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL);
9355 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
9356 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp);
9363 SDValue SRA =
9375 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
9398 SDValue N0 = N->getOperand(0);
9457 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0,
9460 SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0;
9461 SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal;
9462 SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1);
9475 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
9509 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
9511 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
9512 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
9514 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
9521 static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
9525 if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
9539 SDValue N0 = N->getOperand(0);
9544 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
9562 static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
9571 SDValue Op = N->getOperand(0);
9576 SDValue ConstVec = Op->getOperand(1);
9624 SDValue FixConv =
9637 static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
9643 SDValue Op = N->getOperand(0);
9650 SDValue ConstVec = N->getOperand(1);
9691 SDValue ConvInput = Op.getOperand(0);
9706 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,
9706 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,
9728 static SDValue tryCombineToEXTR(SDNode *N,
9739 SDValue LHS;
9745 SDValue RHS;
9768 static SDValue tryCombineToBSL(SDNode *N,
9777 SDValue N0 = N->getOperand(0);
9781 SDValue N1 = N->getOperand(1);
9815 static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
9824 if (SDValue Res = tryCombineToEXTR(N, DCI))
9827 if (SDValue Res = tryCombineToBSL(N, DCI))
9833 static SDValue performANDCombine(SDNode *N,
9836 SDValue LHS = N->getOperand(0);
9853 SDValue NewOp;
9873 static SDValue performSRLCombine(SDNode *N,
9883 SDValue N0 = N->getOperand(0);
9886 SDValue N1 = N->getOperand(1);
9887 SDValue N00 = N0.getOperand(0);
9901 static SDValue performBitcastCombine(SDNode *N,
9925 SDValue Op0 = N->getOperand(0);
9943 SDValue Source = Op0->getOperand(0)->getOperand(0);
9959 SDValue HalfIdx = DAG.getConstant(NumElements, dl, MVT::i64);
9962 SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, dl, MVT::i32);
9969 static SDValue performConcatVectorsCombine(SDNode *N,
9974 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
9990 SDValue N00 = N0->getOperand(0);
9991 SDValue N10 = N1->getOperand(0);
10034 SDValue RHS = N1->getOperand(0);
10051 static SDValue tryCombineFixedPointConvert(SDNode *N,
10068 SDValue Op1 = N->getOperand(1);
10071 SDValue IID = N->getOperand(0);
10072 SDValue Shift = N->getOperand(2);
10073 SDValue Vec = Op1.getOperand(0);
10074 SDValue Lane = Op1.getOperand(1);
10091 SDValue Convert =
10113 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
10113 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
10148 static bool isEssentiallyExtractHighSubvector(SDValue N) {
10159 const SDValue *Opnd0;
10160 const SDValue *Opnd1;
10166 const SDValue *Cmp;
10190 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
10233 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
10246 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
10248 SDValue LHS = Op->getOperand(0);
10249 SDValue RHS = Op->getOperand(1);
10266 SDValue CCVal;
10267 SDValue Cmp;
10296 static SDValue performAddSubLongCombine(SDNode *N,
10310 SDValue LHS = N->getOperand(0);
10311 SDValue RHS = N->getOperand(1);
10345 static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
10351 SDValue LHS = N->getOperand(1);
10352 SDValue RHS = N->getOperand(2);
10374 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
10445 static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
10446 SDValue AndN = N->getOperand(2);
10458 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
10468 static SDValue performIntrinsicCombine(SDNode *N,
10526 static SDValue performExtendCombine(SDNode *N,
10539 SDValue NewABD = tryCombineLongOpWithDup(IID, ABDNode, DCI, DAG);
10585 SDValue Src = N->getOperand(0);
10604 SDValue Lo, Hi;
10624 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
10625 SDValue SplatVal, unsigned NumVecElts) {
10635 SDValue BasePtr = St.getBasePtr();
10639 SDValue NewST1 =
10653 SDValue OffsetPtr =
10679 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
10680 SDValue StVal = St.getValue();
10715 SDValue EltVal = StVal.getOperand(I);
10732 SDValue SplatVal =
10742 static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
10743 SDValue StVal = St.getValue();
10765 SDValue SplatVal;
10795 static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
10803 SDValue StVal = S->getValue();
10811 if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
10842 if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
10850 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
10852 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
10854 SDValue BasePtr = S->getBasePtr();
10855 SDValue NewST1 =
10858 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
10867 static SDValue performPostLD1Combine(SDNode *N,
10883 SDValue Lane;
10907 SDValue Addr = LD->getOperand(1);
10908 SDValue Vector = N->getOperand(0);
10918 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
10939 SmallVector<SDValue, 8> Ops;
10951 SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
10956 SDValue NewResults[] = {
10971 static bool performTBISimplification(SDValue Addr,
10986 static SDValue performSTORECombine(SDNode *N,
10990 if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
11003 static SDValue performNEONPostLDSTCombine(SDNode *N,
11010 SDValue Addr = N->getOperand(AddrOpIdx);
11091 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
11101 SmallVector<SDValue, 8> Ops;
11121 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
11126 std::vector<SDValue> NewResults;
11142 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
11321 SDValue performCONDCombine(SDNode *N,
11352 SDValue AddValue = AndNode->getOperand(0);
11359 SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
11360 SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
11361 SDValue SubsInputValue = SubsNode->getOperand(1);
11386 SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
11388 SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
11395 static SDValue performBRCONDCombine(SDNode *N,
11405 if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
11407 SDValue Chain = N->getOperand(0);
11408 SDValue Dest = N->getOperand(1);
11409 SDValue CCVal = N->getOperand(2);
11410 SDValue Cmp = N->getOperand(3);
11426 SDValue LHS = Cmp.getOperand(0);
11427 SDValue RHS = Cmp.getOperand(1);
11445 SDValue BR;
11461 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
11461 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
11534 static SDValue performTBZCombine(SDNode *N,
11539 SDValue TestSrc = N->getOperand(1);
11540 SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
11565 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
11566 SDValue N0 = N->getOperand(0);
11580 SDValue IfTrue = N->getOperand(1);
11581 SDValue IfFalse = N->getOperand(2);
11582 SDValue SetCC =
11594 static SDValue performSelectCombine(SDNode *N,
11597 SDValue N0 = N->getOperand(0);
11639 SDValue LHS =
11641 SDValue RHS =
11643 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
11647 SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
11655 static SDValue performNVCASTCombine(SDNode *N) {
11665 static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
11710 SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
11715 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
11816 SDValue &Chain) const {
11822 SDValue TCChain = Chain;
11856 bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
11857 SDValue &Offset,
11880 bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
11881 SDValue &Offset,
11885 SDValue Ptr;
11903 SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
11903 SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
11906 SDValue Ptr;
11927 static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
11930 SDValue Op = N->getOperand(0);
11945 SmallVectorImpl<SDValue> &Results,
11949 SDValue Lo, Hi;
11953 SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
11954 SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
11958 static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
11958 static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
11958 static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
11960 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N);
11961 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
11968 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
11968 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
11970 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i64);
11971 SDValue VHi = DAG.getAnyExtOrTrunc(
11976 SDValue RegClass =
11978 SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
11979 SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
11980 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
11986 SmallVectorImpl<SDValue> &Results,
11995 SDValue Ops[] = {
12040 SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
12055 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
12349 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
12349 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
lib/Target/AArch64/AArch64ISelLowering.h 264 void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
277 bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
294 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
294 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
298 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
324 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
324 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
353 bool isZExtFree(SDValue Val, EVT VT2) const override;
490 bool hasAndNotCompare(SDValue V) const override {
495 bool hasAndNot(SDValue Y) const override {
505 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
505 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
583 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
583 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
587 SmallVectorImpl<SDValue> &InVals) const override;
589 SDValue LowerCall(CallLoweringInfo & /*CLI*/,
590 SmallVectorImpl<SDValue> &InVals) const override;
592 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
592 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
592 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
596 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
597 SDValue ThisVal) const;
599 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
599 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
601 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
601 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
604 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
606 const SmallVectorImpl<SDValue> &OutVals,
612 SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,
612 SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,
618 SDValue &Chain) const;
625 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
625 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
627 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
630 SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
632 SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG,
634 SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG,
636 SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
639 SDValue getGOT(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
641 SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
643 SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
645 SDValue getAddrTiny(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
646 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
646 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
647 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
647 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
648 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
648 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
649 SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
649 SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
650 SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
650 SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
651 SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
651 SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
653 SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
653 SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
654 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
654 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
655 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
655 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
656 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
656 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
657 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
657 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
658 SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
658 SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
658 SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
659 SDValue TVal, SDValue FVal, const SDLoc &dl,
659 SDValue TVal, SDValue FVal, const SDLoc &dl,
661 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
661 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
662 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
662 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
663 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
663 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
664 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
664 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
665 SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;
665 SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;
666 SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;
666 SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;
667 SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
667 SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
668 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
668 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
669 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
669 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
670 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
670 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
671 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
671 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
672 SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
672 SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
673 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
673 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
674 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
674 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
675 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
675 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
676 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
676 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
677 SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
677 SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
678 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
678 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
679 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
679 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
680 SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
680 SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
681 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
681 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
682 SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
682 SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
683 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
683 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
684 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
684 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
685 SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
685 SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
686 SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
686 SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
687 SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
687 SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
689 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
689 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
690 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
690 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
691 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
691 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
692 SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
692 SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
693 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
693 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
694 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
694 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
695 SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
695 SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
696 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
696 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
697 SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
697 SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
698 SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
698 SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
699 SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
699 SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
700 SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
700 SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
701 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
701 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
702 SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
702 SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
702 SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
703 SDValue &Size,
706 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
708 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
708 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
711 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
711 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
731 void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
732 std::vector<SDValue> &Ops,
744 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
745 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
747 bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset,
747 bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset,
750 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
750 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
753 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
754 SDValue &Offset, ISD::MemIndexedMode &AM,
757 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
lib/Target/AArch64/AArch64SelectionDAGInfo.cpp 18 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20 SDValue Size, unsigned Align, bool isVolatile,
50 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
50 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
62 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
63 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
63 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
69 SDValue TagSrc = Ptr;
81 SmallVector<SDValue, 8> OutChains;
85 SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
86 SDValue St = DAG.getMemIntrinsicNode(
97 SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
98 SDValue St = DAG.getMemIntrinsicNode(
108 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
112 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
113 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
113 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
114 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
140 SDValue Ops[] = {DAG.getConstant(ObjSize, dl, MVT::i64), Addr, Chain};
lib/Target/AArch64/AArch64SelectionDAGInfo.h 22 SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
23 SDValue Chain, SDValue Dst, SDValue Src,
23 SDValue Chain, SDValue Dst, SDValue Src,
23 SDValue Chain, SDValue Dst, SDValue Src,
24 SDValue Size, unsigned Align, bool isVolatile,
26 SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl,
27 SDValue Chain, SDValue Op1, SDValue Op2,
27 SDValue Chain, SDValue Op1, SDValue Op2,
27 SDValue Chain, SDValue Op1, SDValue Op2,
lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp 73 static bool isNullConstantOrUndef(SDValue V) {
81 static bool getConstantValue(SDValue N, uint32_t &Out) {
162 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
162 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
162 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
163 bool isNoNanSrc(SDValue N) const;
175 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
175 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
176 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
180 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
180 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
180 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
181 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
181 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
181 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
182 bool isDSOffsetLegal(SDValue Base, unsigned Offset,
184 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
184 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
184 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
185 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
185 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
185 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
186 SDValue &Offset1) const;
187 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
187 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
187 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
188 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
188 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
188 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
189 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
189 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
189 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
189 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
190 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
190 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
190 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
191 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
191 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
191 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
192 SDValue &SOffset, SDValue &Offset, SDValue &GLC,
192 SDValue &SOffset, SDValue &Offset, SDValue &GLC,
192 SDValue &SOffset, SDValue &Offset, SDValue &GLC,
193 SDValue &SLC, SDValue &TFE, SDValue &DLC,
193 SDValue &SLC, SDValue &TFE, SDValue &DLC,
193 SDValue &SLC, SDValue &TFE, SDValue &DLC,
194 SDValue &SWZ) const;
195 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
195 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
196 SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
196 SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
196 SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
197 SDValue &SLC) const;
199 SDValue Addr, SDValue &RSrc, SDValue &VAddr,
199 SDValue Addr, SDValue &RSrc, SDValue &VAddr,
199 SDValue Addr, SDValue &RSrc, SDValue &VAddr,
200 SDValue &SOffset, SDValue &ImmOffset) const;
200 SDValue &SOffset, SDValue &ImmOffset) const;
202 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
202 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
202 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
203 SDValue &Offset) const;
205 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
205 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
205 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
206 SDValue &Offset, SDValue &GLC, SDValue &SLC,
206 SDValue &Offset, SDValue &GLC, SDValue &SLC,
206 SDValue &Offset, SDValue &GLC, SDValue &SLC,
207 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
207 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
207 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
208 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
208 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
208 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
209 SDValue &Offset, SDValue &SLC) const;
209 SDValue &Offset, SDValue &SLC) const;
210 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
210 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
210 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
211 SDValue &Offset) const;
214 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
214 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
215 SDValue &Offset, SDValue &SLC) const;
215 SDValue &Offset, SDValue &SLC) const;
216 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr,
216 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr,
217 SDValue &Offset, SDValue &SLC) const;
217 SDValue &Offset, SDValue &SLC) const;
218 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr,
218 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr,
219 SDValue &Offset, SDValue &SLC) const;
219 SDValue &Offset, SDValue &SLC) const;
221 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
221 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
223 SDValue Expand32BitAddress(SDValue Addr) const;
223 SDValue Expand32BitAddress(SDValue Addr) const;
224 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
224 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
224 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
226 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
226 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
226 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
227 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
227 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
227 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
228 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
228 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
228 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
229 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
229 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
230 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
230 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
231 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
231 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
231 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
233 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
233 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
233 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
234 bool SelectVOP3Mods_f32(SDValue In, SDValue &Src, SDValue &SrcMods) const;
234 bool SelectVOP3Mods_f32(SDValue In, SDValue &Src, SDValue &SrcMods) const;
234 bool SelectVOP3Mods_f32(SDValue In, SDValue &Src, SDValue &SrcMods) const;
235 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
235 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
236 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
236 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
236 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
237 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
237 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
238 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
238 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
238 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
239 SDValue &Clamp, SDValue &Omod) const;
239 SDValue &Clamp, SDValue &Omod) const;
240 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
240 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
240 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
241 SDValue &Clamp, SDValue &Omod) const;
241 SDValue &Clamp, SDValue &Omod) const;
243 bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
243 bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
243 bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
244 SDValue &Clamp,
245 SDValue &Omod) const;
247 bool SelectVOP3OMods(SDValue In, SDValue &Src,
247 bool SelectVOP3OMods(SDValue In, SDValue &Src,
248 SDValue &Clamp, SDValue &Omod) const;
248 SDValue &Clamp, SDValue &Omod) const;
250 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
250 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
250 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
251 bool SelectVOP3PMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
251 bool SelectVOP3PMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
251 bool SelectVOP3PMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
252 SDValue &Clamp) const;
254 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
254 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
254 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
255 bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods,
255 bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods,
255 bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods,
256 SDValue &Clamp) const;
258 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
258 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
258 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
259 bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
259 bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
259 bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
260 SDValue &Clamp) const;
261 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
261 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
262 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
262 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
262 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
264 SDValue getHi16Elt(SDValue In) const;
264 SDValue getHi16Elt(SDValue In) const;
266 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
277 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
300 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
300 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
301 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
301 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
302 SDValue& Offset);
309 bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
309 bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
310 SDValue &Offset) override;
311 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
311 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
312 SDValue &Offset) override;
323 static SDValue stripBitcast(SDValue Val) {
323 static SDValue stripBitcast(SDValue Val) {
328 static bool isExtractHiElt(SDValue In, SDValue &Out) {
328 static bool isExtractHiElt(SDValue In, SDValue &Out) {
333 SDValue Srl = In.getOperand(0);
348 static SDValue stripExtractLoElt(SDValue In) {
348 static SDValue stripExtractLoElt(SDValue In) {
350 SDValue Src = In.getOperand(0);
404 SDValue Lo = N->getOperand(0);
405 SDValue Hi = N->getOperand(1);
418 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
419 SDValue Ops[] = {
431 SDValue NewLoadHi =
446 SDValue TiedIn = getHi16Elt(Hi);
461 SDValue Ops[] = {
465 SDValue NewLoadLo =
506 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
581 SDValue SubRegOp = N->getOperand(OpNo + 1);
589 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
590 SDValue Glue) const {
591 SmallVector <SDValue, 8> Ops;
600 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
606 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
632 const SDValue Ops[] = {
668 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
681 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
791 SDValue RC, SubReg0, SubReg1;
804 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
941 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
941 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
942 SDValue &Offset) {
946 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
946 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
947 SDValue &Offset) {
970 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
981 SDValue LHS = N->getOperand(0);
982 SDValue RHS = N->getOperand(1);
990 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
991 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1010 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
1013 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
1016 SDValue AddHiArgs[] = {
1023 SDValue RegSequenceArgs[] = {
1044 SDValue LHS = N->getOperand(0);
1045 SDValue RHS = N->getOperand(1);
1046 SDValue CI = N->getOperand(2);
1071 SDValue Ops[10];
1085 SDValue Ops[8];
1106 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) };
1122 SDValue CarryIn = N->getOperand(3);
1124 SDValue VCC = CurDAG->getCopyToReg(CurDAG->getEntryNode(), SL,
1127 SDValue Ops[10];
1146 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1147 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1152 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset,
1167 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1167 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1168 SDValue &Offset) const {
1171 SDValue N0 = Addr.getOperand(0);
1172 SDValue N1 = Addr.getOperand(1);
1185 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1190 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1194 SmallVector<SDValue, 3> Opnds;
1224 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1240 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1240 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1241 SDValue &Offset0,
1242 SDValue &Offset1) const {
1246 SDValue N0 = Addr.getOperand(0);
1247 SDValue N1 = Addr.getOperand(1);
1266 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1271 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1275 SmallVector<SDValue, 3> Opnds;
1301 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1320 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
1320 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
1321 SDValue &VAddr, SDValue &SOffset,
1321 SDValue &VAddr, SDValue &SOffset,
1322 SDValue &Offset, SDValue &Offen,
1322 SDValue &Offset, SDValue &Offen,
1323 SDValue &Idxen, SDValue &Addr64,
1323 SDValue &Idxen, SDValue &Addr64,
1324 SDValue &GLC, SDValue &SLC,
1324 SDValue &GLC, SDValue &SLC,
1325 SDValue &TFE, SDValue &DLC,
1325 SDValue &TFE, SDValue &DLC,
1326 SDValue &SWZ) const {
1347 SDValue N0 = Addr;
1359 SDValue N2 = N0.getOperand(0);
1360 SDValue N3 = N0.getOperand(1);
1415 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1415 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1416 SDValue &VAddr, SDValue &SOffset,
1416 SDValue &VAddr, SDValue &SOffset,
1417 SDValue &Offset, SDValue &GLC,
1417 SDValue &Offset, SDValue &GLC,
1418 SDValue &SLC, SDValue &TFE,
1418 SDValue &SLC, SDValue &TFE,
1419 SDValue &DLC, SDValue &SWZ) const {
1419 SDValue &DLC, SDValue &SWZ) const {
1420 SDValue Ptr, Offen, Idxen, Addr64;
1444 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1444 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1445 SDValue &VAddr, SDValue &SOffset,
1445 SDValue &VAddr, SDValue &SOffset,
1446 SDValue &Offset,
1447 SDValue &SLC) const {
1449 SDValue GLC, TFE, DLC, SWZ;
1459 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1459 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1459 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1464 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1480 SDValue Addr, SDValue &Rsrc,
1480 SDValue Addr, SDValue &Rsrc,
1481 SDValue &VAddr, SDValue &SOffset,
1481 SDValue &VAddr, SDValue &SOffset,
1482 SDValue &ImmOffset) const {
1493 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1512 SDValue N0 = Addr.getOperand(0);
1513 SDValue N1 = Addr.getOperand(1);
1547 SDValue Addr,
1548 SDValue &SRsrc,
1549 SDValue &SOffset,
1550 SDValue &Offset) const {
1573 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1573 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1574 SDValue &SOffset, SDValue &Offset,
1574 SDValue &SOffset, SDValue &Offset,
1575 SDValue &GLC, SDValue &SLC,
1575 SDValue &GLC, SDValue &SLC,
1576 SDValue &TFE, SDValue &DLC,
1576 SDValue &TFE, SDValue &DLC,
1577 SDValue &SWZ) const {
1578 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1602 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1602 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1603 SDValue &Soffset, SDValue &Offset
1603 SDValue &Soffset, SDValue &Offset
1605 SDValue GLC, SLC, TFE, DLC, SWZ;
1609 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1609 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1610 SDValue &Soffset, SDValue &Offset,
1610 SDValue &Soffset, SDValue &Offset,
1611 SDValue &SLC) const {
1612 SDValue GLC, TFE, DLC, SWZ;
1624 for (SDValue V : N->op_values())
1633 SDValue Addr,
1634 SDValue &VAddr,
1635 SDValue &Offset,
1636 SDValue &SLC) const {
1643 SDValue N0 = Addr.getOperand(0);
1644 SDValue N1 = Addr.getOperand(1);
1688 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1689 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1696 SDValue AddOffsetLo
1698 SDValue AddOffsetHi
1702 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1712 SDValue RegSequenceArgs[] = {
1729 SDValue Addr,
1730 SDValue &VAddr,
1731 SDValue &Offset,
1732 SDValue &SLC) const {
1737 SDValue Addr,
1738 SDValue &VAddr,
1739 SDValue &Offset,
1740 SDValue &SLC) const {
1744 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1745 SDValue &Offset, bool &Imm) const {
1770 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1778 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1778 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1788 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1790 const SDValue Ops[] = {
1803 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1803 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1804 SDValue &Offset, bool &Imm) const {
1812 SDValue N0 = Addr.getOperand(0);
1813 SDValue N1 = Addr.getOperand(1);
1826 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
1826 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
1827 SDValue &Offset) const {
1832 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
1832 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
1833 SDValue &Offset) const {
1845 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
1845 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
1846 SDValue &Offset) const {
1852 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
1853 SDValue &Offset) const {
1858 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
1859 SDValue &Offset) const {
1870 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
1871 SDValue &Base,
1872 SDValue &Offset) const {
1876 SDValue N0 = Index.getOperand(0);
1877 SDValue N1 = Index.getOperand(1);
1899 SDValue Val, uint32_t Offset,
1905 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
1915 const SDValue &Shl = N->getOperand(0);
1941 const SDValue &Srl = N->getOperand(0);
1963 const SDValue &And = N->getOperand(0);
1993 SDValue Src = N->getOperand(0);
2016 SDValue Cond = N->getOperand(1);
2038 SDValue Cond = N->getOperand(1);
2078 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2095 SDValue Src0 = N->getOperand(0);
2096 SDValue Src1 = N->getOperand(1);
2097 SDValue Src2 = N->getOperand(2);
2113 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2114 SDValue Ops[] = {
2146 SDValue SRsrc, VAddr, SOffset, Offset, SLC;
2151 SDValue CmpVal = Mem->getOperand(2);
2155 SDValue Ops[] = {
2164 SDValue SRsrc, SOffset, Offset, SLC;
2169 SDValue CmpVal = Mem->getOperand(2);
2170 SDValue Ops[] = {
2187 SDValue Extract
2201 SDValue Chain = N->getOperand(0);
2202 SDValue Ptr = N->getOperand(2);
2207 SDValue Offset;
2209 SDValue PtrBase = Ptr.getOperand(0);
2210 SDValue PtrOffset = Ptr.getOperand(1);
2224 SDValue Ops[] = {
2267 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2305 SDValue Chain = N->getOperand(0);
2306 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2309 SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1);
2312 SmallVector<SDValue, 5> Ops;
2356 SDValue Src = N->getOperand(1);
2378 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2378 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2396 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2396 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2397 SDValue &SrcMods) const {
2407 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2407 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2408 SDValue &SrcMods) const {
2413 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_f32(SDValue In, SDValue &Src,
2413 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_f32(SDValue In, SDValue &Src,
2414 SDValue &SrcMods) const {
2422 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2422 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2430 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2430 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2431 SDValue &SrcMods, SDValue &Clamp,
2431 SDValue &SrcMods, SDValue &Clamp,
2432 SDValue &Omod) const {
2440 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
2440 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
2441 SDValue &SrcMods,
2442 SDValue &Clamp,
2443 SDValue &Omod) const {
2448 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2448 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2449 SDValue &Clamp, SDValue &Omod) const {
2449 SDValue &Clamp, SDValue &Omod) const {
2459 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2459 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2460 SDValue &SrcMods) const {
2472 SDValue Lo = stripBitcast(Src.getOperand(0));
2473 SDValue Hi = stripBitcast(Src.getOperand(1));
2513 bool AMDGPUDAGToDAGISel::SelectVOP3PMods0(SDValue In, SDValue &Src,
2513 bool AMDGPUDAGToDAGISel::SelectVOP3PMods0(SDValue In, SDValue &Src,
2514 SDValue &SrcMods,
2515 SDValue &Clamp) const {
2524 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2524 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2525 SDValue &SrcMods) const {
2532 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel0(SDValue In, SDValue &Src,
2532 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel0(SDValue In, SDValue &Src,
2533 SDValue &SrcMods,
2534 SDValue &Clamp) const {
2543 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2543 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2544 SDValue &SrcMods) const {
2549 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src,
2549 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src,
2550 SDValue &SrcMods,
2551 SDValue &Clamp) const {
2562 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2562 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2603 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2603 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2604 SDValue &SrcMods) const {
2611 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2611 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2626 SDValue Src;
2747 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
2748 SDValue& IntPtr) {
2757 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
2758 SDValue& BaseReg, SDValue &Offset) {
2758 SDValue& BaseReg, SDValue &Offset) {
2804 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
2804 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
2805 SDValue &Offset) {
2828 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
2828 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
2829 SDValue &Offset) {
lib/Target/AMDGPU/AMDGPUISelLowering.cpp 51 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
57 unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
819 bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1019 SDValue AMDGPUTargetLowering::LowerReturn(
1020 SDValue Chain, CallingConv::ID CallConv,
1023 const SmallVectorImpl<SDValue> &OutVals,
1046 SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1046 SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1050 SmallVector<SDValue, 8> ArgChains;
1082 SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1083 SmallVectorImpl<SDValue> &InVals,
1085 SDValue Callee = CLI.Callee;
1109 SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1110 SmallVectorImpl<SDValue> &InVals) const {
1114 SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1114 SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1125 SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1125 SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1167 SmallVectorImpl<SDValue> &Results,
1191 SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
1192 SDValue Op,
1226 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1226 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1228 SmallVector<SDValue, 8> Args;
1233 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0));
1234 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1));
1236 SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi });
1246 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1246 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1249 SmallVector<SDValue, 8> Args;
1259 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
1260 SDValue LHS, SDValue RHS,
1260 SDValue LHS, SDValue RHS,
1261 SDValue True, SDValue False,
1261 SDValue True, SDValue False,
1262 SDValue CC,
1332 std::pair<SDValue, SDValue>
1332 std::pair<SDValue, SDValue>
1333 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1336 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1338 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1339 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1341 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1342 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1347 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1347 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1350 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1351 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1355 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1355 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1358 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1359 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1381 std::pair<SDValue, SDValue>
1381 std::pair<SDValue, SDValue>
1382 AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1390 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
1392 SDValue Hi = DAG.getNode(
1398 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1398 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1409 SDValue BasePtr = Load->getBasePtr();
1417 SDValue Lo, Hi;
1427 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1430 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, Size);
1431 SDValue HiLoad =
1437 SDValue Join;
1450 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1457 SDValue AMDGPUTargetLowering::WidenVectorLoad(SDValue Op,
1457 SDValue AMDGPUTargetLowering::WidenVectorLoad(SDValue Op,
1462 SDValue BasePtr = Load->getBasePtr();
1472 SDValue WideLoad = DAG.getExtLoad(
1482 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1482 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1485 SDValue Val = Store->getValue();
1494 SDValue Chain = Store->getChain();
1495 SDValue BasePtr = Store->getBasePtr();
1500 SDValue Lo, Hi;
1506 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());
1513 SDValue LoStore =
1516 SDValue HiStore =
1526 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
1526 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
1530 SDValue LHS = Op.getOperand(0);
1531 SDValue RHS = Op.getOperand(1);
1552 SDValue jq = DAG.getConstant(1, DL, IntVT);
1567 SDValue ia = LHS;
1570 SDValue ib = RHS;
1573 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
1576 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
1578 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
1585 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
1591 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
1594 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
1605 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
1611 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
1614 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
1619 SDValue InRegSize
1624 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
1632 void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
1634 SmallVectorImpl<SDValue> &Results) const {
1642 SDValue One = DAG.getConstant(1, DL, HalfVT);
1643 SDValue Zero = DAG.getConstant(0, DL, HalfVT);
1646 SDValue LHS = Op.getOperand(0);
1647 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1648 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One);
1650 SDValue RHS = Op.getOperand(1);
1651 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1652 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One);
1657 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1660 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
1661 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
1674 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
1675 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
1676 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
1679 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
1680 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
1682 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
1684 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
1685 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
1688 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
1689 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
1690 SDValue Rcp64 = DAG.getBitcast(VT,
1693 SDValue Zero64 = DAG.getConstant(0, DL, VT);
1694 SDValue One64 = DAG.getConstant(1, DL, VT);
1695 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
1698 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
1699 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
1700 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
1701 SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1703 SDValue Mulhi1_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1706 SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
1708 SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
1710 SDValue Add1_HiNc = DAG.getNode(ISD::ADD, DL, HalfVT, Rcp_Hi, Mulhi1_Hi);
1711 SDValue Add1 = DAG.getBitcast(VT,
1714 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
1715 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
1716 SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1718 SDValue Mulhi2_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1721 SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
1723 SDValue Add2_HiC = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_HiNc,
1725 SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add2_HiC,
1727 SDValue Add2 = DAG.getBitcast(VT,
1729 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
1731 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
1733 SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero);
1734 SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One);
1735 SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
1737 SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
1739 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
1740 SDValue Sub1 = DAG.getBitcast(VT,
1743 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
1744 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
1746 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
1748 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
1755 SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
1757 SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
1759 SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1761 SDValue Sub2 = DAG.getBitcast(VT,
1764 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
1766 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
1768 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
1770 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
1773 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
1775 SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
1777 SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1779 SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
1781 SDValue Sub3 = DAG.getBitcast(VT,
1787 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
1788 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
1790 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
1791 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
1801 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
1802 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
1804 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
1805 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
1808 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
1809 SDValue DIV_Lo = Zero;
1815 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
1817 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
1826 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
1827 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
1832 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
1836 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
1842 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
1842 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
1848 SmallVector<SDValue, 2> Results;
1854 if (SDValue Res = LowerDIVREM24(Op, DAG, false))
1858 SDValue Num = Op.getOperand(0);
1859 SDValue Den = Op.getOperand(1);
1863 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1866 SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den);
1869 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1872 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
1876 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
1881 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1884 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1887 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1890 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
1894 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1897 SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den);
1900 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1903 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
1908 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1914 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1920 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
1924 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
1928 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
1938 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1941 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1944 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
1950 SDValue Ops[2] = {
1957 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
1957 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
1962 SDValue LHS = Op.getOperand(0);
1963 SDValue RHS = Op.getOperand(1);
1965 SDValue Zero = DAG.getConstant(0, DL, VT);
1966 SDValue NegOne = DAG.getConstant(-1, DL, VT);
1969 if (SDValue Res = LowerDIVREM24(Op, DAG, true))
1979 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1980 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1981 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1983 SDValue Res[2] = {
1990 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
1991 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
1992 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
1993 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2001 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
2002 SDValue Rem = Div.getValue(1);
2010 SDValue Res[2] = {
2018 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
2018 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
2021 SDValue X = Op.getOperand(0);
2022 SDValue Y = Op.getOperand(1);
2026 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
2027 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
2028 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
2033 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2033 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2035 SDValue Src = Op.getOperand(0);
2041 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2043 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2044 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
2049 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
2050 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2051 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2053 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
2058 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2058 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2063 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
2067 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
2073 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2073 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2075 SDValue Src = Op.getOperand(0);
2079 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2080 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2082 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2086 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
2088 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2093 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
2094 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
2097 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
2100 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
2101 const SDValue FractMask
2104 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
2105 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
2106 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
2111 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
2113 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2114 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2116 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
2117 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
2122 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2122 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2124 SDValue Src = Op.getOperand(0);
2129 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
2130 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
2134 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
2135 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
2137 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
2140 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
2144 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
2149 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
2149 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
2161 SDValue AMDGPUTargetLowering::LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const {
2161 SDValue AMDGPUTargetLowering::LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const {
2163 SDValue X = Op.getOperand(0);
2166 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2170 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2172 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2174 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2175 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2176 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2178 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);
2183 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
2185 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);
2190 SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const {
2190 SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const {
2192 SDValue X = Op.getOperand(0);
2194 SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X);
2196 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2197 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2198 const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32);
2199 const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32);
2203 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
2205 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One);
2207 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2209 const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), SL,
2212 SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp);
2213 SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64,
2218 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M);
2219 SDValue Tmp1 = DAG.getSetCC(SL, SetCCVT,
2223 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1,
2225 SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2);
2230 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2231 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2232 SDValue ExpEqNegOne = DAG.getSetCC(SL, SetCCVT, NegOne, Exp, ISD::SETEQ);
2234 SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64,
2239 SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X);
2247 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2247 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2259 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2259 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2261 SDValue Src = Op.getOperand(0);
2267 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2269 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2270 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
2275 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
2276 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2277 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2279 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
2284 SDValue AMDGPUTargetLowering::LowerFLOG(SDValue Op, SelectionDAG &DAG,
2284 SDValue AMDGPUTargetLowering::LowerFLOG(SDValue Op, SelectionDAG &DAG,
2289 SDValue Operand = Op.getOperand(0);
2290 SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand);
2291 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2297 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
2297 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
2300 SDValue Src = Op.getOperand(0);
2302 const SDValue K = DAG.getConstantFP(numbers::log2e, SL, VT);
2303 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags());
2315 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
2315 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
2317 SDValue Src = Op.getOperand(0);
2335 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2337 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2338 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2340 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
2341 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
2346 SDValue HiOrLo = isCtlzOpc(Op.getOpcode()) ? Hi : Lo;
2347 SDValue Hi0orLo0 = DAG.getSetCC(SL, SetCCVT, HiOrLo, Zero, ISD::SETEQ);
2349 SDValue OprLo = DAG.getNode(ISDOpc, SL, MVT::i32, Lo);
2350 SDValue OprHi = DAG.getNode(ISDOpc, SL, MVT::i32, Hi);
2352 const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32);
2353 SDValue Add, NewOpr;
2369 SDValue LoOrHi = isCtlzOpc(Op.getOpcode()) ? Lo : Hi;
2370 SDValue Lo0OrHi0 = DAG.getSetCC(SL, SetCCVT, LoOrHi, Zero, ISD::SETEQ);
2371 SDValue SrcIsZero = DAG.getNode(ISD::AND, SL, SetCCVT, Lo0OrHi0, Hi0orLo0);
2378 const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32);
2389 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
2389 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
2411 SDValue Src = Op.getOperand(0);
2412 SDValue L = Src;
2414 SDValue S;
2416 const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64);
2419 SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S);
2427 SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32);
2428 SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64);
2429 SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L);
2432 SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32);
2433 SDValue E = DAG.getSelect(SL, MVT::i32,
2438 SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64,
2442 SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U,
2445 SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64,
2448 SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32,
2452 SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64);
2453 SDValue RCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETUGT);
2454 SDValue TCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETEQ);
2456 SDValue One = DAG.getConstant(1, SL, MVT::i32);
2458 SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One);
2460 SDValue R = DAG.getSelect(SL, MVT::i32,
2470 SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R);
2474 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
2474 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
2477 SDValue Src = Op.getOperand(0);
2479 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2481 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
2483 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
2486 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
2489 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
2491 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
2497 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
2497 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
2507 SDValue Src = Op.getOperand(0);
2509 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2510 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2511 SDValue FPRound =
2524 SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
2524 SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
2534 SDValue Src = Op.getOperand(0);
2536 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2537 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2538 SDValue FPRound =
2551 SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
2551 SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
2555 SDValue Src = Op.getOperand(0);
2557 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2559 SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
2561 SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
2564 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
2566 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul);
2569 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc);
2571 SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL,
2573 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
2575 SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi});
2580 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
2580 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
2582 SDValue N0 = Op.getOperand(0);
2599 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
2600 SDValue One = DAG.getConstant(1, DL, MVT::i32);
2601 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
2602 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
2606 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2615 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2620 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
2624 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
2628 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
2633 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2638 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
2640 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
2644 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2647 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
2648 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
2649 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
2652 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
2653 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
2657 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
2659 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
2670 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2679 SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
2679 SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
2681 SDValue Src = Op.getOperand(0);
2689 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2690 SDValue FpToInt32 =
2702 SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
2702 SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
2704 SDValue Src = Op.getOperand(0);
2712 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2713 SDValue FpToInt32 =
2725 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2725 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2733 SDValue Src = Op.getOperand(0);
2738 SmallVector<SDValue, 8> Args;
2741 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
2752 static bool isU24(SDValue Op, SelectionDAG &DAG) {
2756 static bool isI24(SDValue Op, SelectionDAG &DAG) {
2763 static SDValue simplifyI24(SDNode *Node24,
2768 SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0);
2769 SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1);
2782 SDValue DemandedLHS = DAG.GetDemandedBits(LHS, Demanded);
2783 SDValue DemandedRHS = DAG.GetDemandedBits(RHS, Demanded);
2801 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
2844 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
2871 SDValue Ops[2];
2885 SDValue NewLoad
2889 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
2896 SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
2935 SDValue Val = SN->getValue();
2940 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
2942 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
2953 SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
2956 SDValue N0 = N->getOperand(0);
2961 SDValue N1 = N->getOperand(1);
2965 SDValue Src = N0.getOperand(0);
2968 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
2976 SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
2990 SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
2992 unsigned Opc, SDValue LHS,
2995 SDValue Lo, Hi;
2998 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
2999 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
3001 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
3002 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
3009 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
3013 SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
3021 SDValue LHS = N->getOperand(0);
3035 SDValue X = LHS->getOperand(0);
3041 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
3054 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
3070 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
3072 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
3073 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
3075 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
3077 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
3081 SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
3096 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
3097 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
3100 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
3106 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
3107 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
3109 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
3116 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
3123 SDValue LHS = N->getOperand(0);
3151 SDValue One = DAG.getConstant(1, SL, MVT::i32);
3152 SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
3154 SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, LHS);
3155 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecOp, One);
3157 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
3158 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
3160 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
3165 SDValue AMDGPUTargetLowering::performTruncateCombine(
3170 SDValue Src = N->getOperand(0);
3174 SDValue Vec = Src.getOperand(0);
3176 SDValue Elt0 = Vec.getOperand(0);
3195 SDValue BV = stripBitcast(Src.getOperand(0));
3198 SDValue SrcElt = BV.getOperand(1);
3221 SDValue Amt = Src.getOperand(1);
3231 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
3240 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
3254 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
3255 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
3255 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
3267 SDValue Mul = DAG.getNode(MulOpc, SL,
3274 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
3289 SDValue N0 = N->getOperand(0);
3290 SDValue N1 = N->getOperand(1);
3302 SDValue Mul;
3321 SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
3331 SDValue N0 = N->getOperand(0);
3332 SDValue N1 = N->getOperand(1);
3340 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
3345 SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
3355 SDValue N0 = N->getOperand(0);
3356 SDValue N1 = N->getOperand(1);
3364 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
3369 SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
3374 if (SDValue V = simplifyI24(N, DCI))
3377 SDValue N0 = N->getOperand(0);
3378 SDValue N1 = N->getOperand(1);
3387 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
3388 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
3392 static bool isNegativeOne(SDValue Val) {
3398 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
3399 SDValue Op,
3411 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
3425 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
3425 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
3426 SDValue LHS, SDValue RHS,
3426 SDValue LHS, SDValue RHS,
3434 SDValue CmpLHS = Cond.getOperand(0);
3460 static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
3463 SDValue Cond,
3464 SDValue N1,
3465 SDValue N2) {
3469 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
3482 static SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
3483 SDValue N) {
3485 SDValue Cond = N.getOperand(0);
3486 SDValue LHS = N.getOperand(1);
3487 SDValue RHS = N.getOperand(2);
3508 SDValue NewLHS = LHS.getOperand(0);
3509 SDValue NewRHS = RHS;
3531 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
3542 SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
3544 if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
3547 SDValue Cond = N->getOperand(0);
3552 SDValue LHS = Cond.getOperand(0);
3553 SDValue RHS = Cond.getOperand(1);
3554 SDValue CC = Cond.getOperand(2);
3556 SDValue True = N->getOperand(1);
3557 SDValue False = N->getOperand(2);
3571 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
3576 SDValue MinMax
3600 bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
3631 SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
3634 SDValue N0 = N->getOperand(0);
3661 SDValue LHS = N0.getOperand(0);
3662 SDValue RHS = N0.getOperand(1);
3674 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
3685 SDValue LHS = N0.getOperand(0);
3686 SDValue RHS = N0.getOperand(1);
3695 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
3708 SDValue LHS = N0.getOperand(0);
3709 SDValue MHS = N0.getOperand(1);
3710 SDValue RHS = N0.getOperand(2);
3724 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
3742 SDValue LHS = N0.getOperand(0);
3743 SDValue RHS = N0.getOperand(1);
3750 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3751 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3754 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
3762 SDValue Ops[3];
3766 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
3783 SDValue CvtSrc = N0.getOperand(0);
3795 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3799 SDValue CvtSrc = N0.getOperand(0);
3811 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3820 SDValue Src = N0.getOperand(0);
3824 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
3833 SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
3836 SDValue N0 = N->getOperand(0);
3845 SDValue Src = N0.getOperand(0);
3849 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
3858 SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
3870 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
3887 SDValue Src = N->getOperand(0);
3895 SmallVector<SDValue, 8> CastedElts;
3898 SDValue Elt = Src.getOperand(I);
3914 SDValue Src = N->getOperand(0);
3919 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3930 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3969 if (SDValue V = simplifyI24(N, DCI))
3998 SDValue BitsFrom = N->getOperand(0);
4044 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
4086 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
4122 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
4131 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
4138 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
4140 SDValue Chain,
4141 SDValue ArgVal,
4146 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
4147 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, 4,
4152 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
4158 SDValue V = Arg.isRegister() ?
4354 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
4354 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
4372 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
4372 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
4394 const SDValue Op, KnownBits &Known,
4536 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4577 bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
lib/Target/AMDGPU/AMDGPUISelLowering.h 36 SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL, unsigned Opc) const;
36 SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL, unsigned Opc) const;
39 static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG);
40 static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG);
44 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
44 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
45 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
45 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
49 SDValue LowerFREM(SDValue Op, SelectionDAG &DAG) const;
49 SDValue LowerFREM(SDValue Op, SelectionDAG &DAG) const;
50 SDValue LowerFCEIL(SDValue Op, SelectionDAG &DAG) const;
50 SDValue LowerFCEIL(SDValue Op, SelectionDAG &DAG) const;
51 SDValue LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const;
51 SDValue LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const;
52 SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const;
52 SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const;
53 SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const;
53 SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const;
55 SDValue LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const;
55 SDValue LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const;
56 SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const;
56 SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const;
57 SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const;
57 SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const;
58 SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const;
58 SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const;
59 SDValue LowerFLOG(SDValue Op, SelectionDAG &DAG,
59 SDValue LowerFLOG(SDValue Op, SelectionDAG &DAG,
61 SDValue lowerFEXP(SDValue Op, SelectionDAG &DAG) const;
61 SDValue lowerFEXP(SDValue Op, SelectionDAG &DAG) const;
63 SDValue LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const;
63 SDValue LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const;
65 SDValue LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, bool Signed) const;
65 SDValue LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, bool Signed) const;
66 SDValue LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, bool Signed) const;
66 SDValue LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, bool Signed) const;
67 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
67 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
68 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
68 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
70 SDValue LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, bool Signed) const;
70 SDValue LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, bool Signed) const;
71 SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const;
71 SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const;
72 SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const;
72 SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const;
73 SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
73 SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
75 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
75 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
79 SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const;
80 SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const;
81 SDValue performAssertSZExtCombine(SDNode *N, DAGCombinerInfo &DCI) const;
82 SDValue performIntrinsicWOChainCombine(SDNode *N, DAGCombinerInfo &DCI) const;
84 SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL,
85 unsigned Opc, SDValue LHS,
87 SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
88 SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const;
89 SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
90 SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const;
91 SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
92 SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
93 SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
94 SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const;
95 SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
95 SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
95 SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
96 SDValue RHS, DAGCombinerInfo &DCI) const;
97 SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
99 bool isConstantCostlierToNegate(SDValue N) const;
100 SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
101 SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
102 SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
106 virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
106 virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
110 std::pair<SDValue, SDValue> split64BitValue(SDValue Op,
110 std::pair<SDValue, SDValue> split64BitValue(SDValue Op,
110 std::pair<SDValue, SDValue> split64BitValue(SDValue Op,
112 SDValue getLoHalf64(SDValue Op, SelectionDAG &DAG) const;
112 SDValue getLoHalf64(SDValue Op, SelectionDAG &DAG) const;
113 SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const;
113 SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const;
122 std::pair<SDValue, SDValue> splitVector(const SDValue &N, const SDLoc &DL,
122 std::pair<SDValue, SDValue> splitVector(const SDValue &N, const SDLoc &DL,
122 std::pair<SDValue, SDValue> splitVector(const SDValue &N, const SDLoc &DL,
127 SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const;
127 SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const;
130 SDValue WidenVectorLoad(SDValue Op, SelectionDAG &DAG) const;
130 SDValue WidenVectorLoad(SDValue Op, SelectionDAG &DAG) const;
133 SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const;
133 SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const;
135 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
135 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
136 SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
136 SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
137 SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
137 SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
138 SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const;
138 SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const;
139 void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG,
140 SmallVectorImpl<SDValue> &Results) const;
149 bool mayIgnoreSignedZero(SDValue Op) const {
160 static inline SDValue stripBitcast(SDValue Val) {
160 static inline SDValue stripBitcast(SDValue Val) {
173 bool isZExtFree(SDValue Val, EVT VT2) const override;
201 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
201 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
203 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
206 SDValue addTokenForArgument(SDValue Chain,
206 SDValue addTokenForArgument(SDValue Chain,
211 SDValue lowerUnhandledCall(CallLoweringInfo &CLI,
212 SmallVectorImpl<SDValue> &InVals,
214 SDValue LowerCall(CallLoweringInfo &CLI,
215 SmallVectorImpl<SDValue> &InVals) const override;
217 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op,
217 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op,
220 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
220 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
221 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
223 SmallVectorImpl<SDValue> &Results,
226 SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS,
226 SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS,
227 SDValue RHS, SDValue True, SDValue False,
227 SDValue RHS, SDValue True, SDValue False,
227 SDValue RHS, SDValue True, SDValue False,
228 SDValue CC, DAGCombinerInfo &DCI) const;
242 bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override {
245 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
245 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
248 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
248 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
257 void computeKnownBitsForTargetNode(const SDValue Op,
263 unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts,
267 bool isKnownNeverNaNForTargetNode(SDValue Op,
277 SDValue CreateLiveInRegister(SelectionDAG &DAG,
282 SDValue CreateLiveInRegister(SelectionDAG &DAG,
289 SDValue CreateLiveInRegisterRaw(SelectionDAG &DAG,
297 SDValue loadStackInputValue(SelectionDAG &DAG,
302 SDValue storeStackInputValue(SelectionDAG &DAG,
304 SDValue Chain,
305 SDValue ArgVal,
308 SDValue loadInputValue(SelectionDAG &DAG,
lib/Target/AMDGPU/R600ISelLowering.cpp 473 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
473 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
490 SDValue Result = LowerLOAD(Op, DAG);
501 SDValue Chain = Op.getOperand(0);
507 const SDValue Args[8] = {
546 SDValue TexArgs[19] = {
570 SDValue Args[8] = {
651 SmallVectorImpl<SDValue> &Results,
672 SDValue Result;
678 SDValue Op = SDValue(N, 1);
679 SDValue RES = LowerSDIVREM(Op, DAG);
685 SDValue Op = SDValue(N, 0);
692 SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG,
693 SDValue Vector) const {
697 SmallVector<SDValue, 8> Args;
708 SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
708 SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
711 SDValue Vector = Op.getOperand(0);
712 SDValue Index = Op.getOperand(1);
723 SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
723 SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
726 SDValue Vector = Op.getOperand(0);
727 SDValue Value = Op.getOperand(1);
728 SDValue Index = Op.getOperand(2);
735 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(),
740 SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
741 SDValue Op,
751 SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT);
755 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
755 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
759 SDValue Arg = Op.getOperand(0);
763 SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT,
779 SDValue TrigVal = DAG.getNode(TrigNode, DL, VT,
789 SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const {
789 SDValue R600TargetLowering::LowerSHLParts(SDValue Op, SelectionDAG &DAG) const {
793 SDValue Lo = Op.getOperand(0);
794 SDValue Hi = Op.getOperand(1);
795 SDValue Shift = Op.getOperand(2);
796 SDValue Zero = DAG.getConstant(0, DL, VT);
797 SDValue One = DAG.getConstant(1, DL, VT);
799 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT);
800 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
801 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width);
802 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift);
809 SDValue Overflow = DAG.getNode(ISD::SRL, DL, VT, Lo, CompShift);
812 SDValue HiSmall = DAG.getNode(ISD::SHL, DL, VT, Hi, Shift);
814 SDValue LoSmall = DAG.getNode(ISD::SHL, DL, VT, Lo, Shift);
816 SDValue HiBig = DAG.getNode(ISD::SHL, DL, VT, Lo, BigShift);
817 SDValue LoBig = Zero;
825 SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const {
825 SDValue R600TargetLowering::LowerSRXParts(SDValue Op, SelectionDAG &DAG) const {
829 SDValue Lo = Op.getOperand(0);
830 SDValue Hi = Op.getOperand(1);
831 SDValue Shift = Op.getOperand(2);
832 SDValue Zero = DAG.getConstant(0, DL, VT);
833 SDValue One = DAG.getConstant(1, DL, VT);
837 SDValue Width = DAG.getConstant(VT.getSizeInBits(), DL, VT);
838 SDValue Width1 = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
839 SDValue BigShift = DAG.getNode(ISD::SUB, DL, VT, Shift, Width);
840 SDValue CompShift = DAG.getNode(ISD::SUB, DL, VT, Width1, Shift);
847 SDValue Overflow = DAG.getNode(ISD::SHL, DL, VT, Hi, CompShift);
850 SDValue HiSmall = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, Shift);
851 SDValue LoSmall = DAG.getNode(ISD::SRL, DL, VT, Lo, Shift);
854 SDValue LoBig = DAG.getNode(SRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, BigShift);
855 SDValue HiBig = SRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, Width1) : Zero;
863 SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
863 SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
868 SDValue Lo = Op.getOperand(0);
869 SDValue Hi = Op.getOperand(1);
871 SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi);
876 SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi);
881 SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const {
881 SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const {
891 SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const {
891 SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const {
901 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
916 bool R600TargetLowering::isZero(SDValue Op) const {
926 bool R600TargetLowering::isHWTrueValue(SDValue Op) const {
933 bool R600TargetLowering::isHWFalseValue(SDValue Op) const {
940 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
940 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
944 SDValue LHS = Op.getOperand(0);
945 SDValue RHS = Op.getOperand(1);
946 SDValue True = Op.getOperand(2);
947 SDValue False = Op.getOperand(3);
948 SDValue CC = Op.getOperand(4);
949 SDValue Temp;
953 SDValue MinMax = combineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI);
1026 SDValue Cond = LHS;
1027 SDValue Zero = RHS;
1050 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT,
1059 SDValue HWTrue, HWFalse;
1074 SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC);
1087 SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
1087 SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
1138 SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store,
1146 SDValue Mask;
1157 SDValue OldChain = Store->getChain();
1160 SDValue Chain = VectorTrunc ? OldChain->getOperand(0) : OldChain;
1161 SDValue BasePtr = Store->getBasePtr();
1162 SDValue Offset = Store->getOffset();
1165 SDValue LoadPtr = BasePtr;
1172 SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1179 SDValue Dst = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1184 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1188 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1193 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
1197 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT);
1200 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
1204 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, Mask, ShiftAmt);
1214 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
1218 SDValue NewStore = DAG.getStore(Chain, DL, Value, Ptr, PtrInfo);
1229 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1229 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1233 SDValue Chain = StoreNode->getChain();
1234 SDValue Ptr = StoreNode->getBasePtr();
1235 SDValue Value = StoreNode->getValue();
1251 SDValue NewChain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, Chain);
1253 SDValue NewStore = DAG.getTruncStore(
1270 SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, PtrVT, Ptr,
1278 SDValue MaskConstant;
1287 SDValue ByteIndex = DAG.getNode(ISD::AND, DL, PtrVT, Ptr,
1289 SDValue BitShift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex,
1293 SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, BitShift);
1296 SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant);
1297 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, BitShift);
1301 SDValue Src[4] = {
1307 SDValue Input = DAG.getBuildVector(MVT::v4i32, DL, Src);
1308 SDValue Args[3] = { Chain, Input, DWordAddr };
1384 SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op,
1384 SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op,
1392 SDValue BasePtr = Load->getBasePtr();
1393 SDValue Chain = Load->getChain();
1394 SDValue Offset = Load->getOffset();
1396 SDValue LoadPtr = BasePtr;
1403 SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1410 SDValue Read = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1413 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
1417 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1421 SDValue Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Read, ShiftAmt);
1427 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1433 SDValue Ops[] = {
1441 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1441 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1454 SDValue Chain = LoadNode->getChain();
1455 SDValue Ptr = LoadNode->getBasePtr();
1468 SDValue Result;
1488 SDValue MergedValues[2] = {
1505 SDValue NewLoad = DAG.getExtLoad(
1508 SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad,
1511 SDValue MergedValues[2] = { Res, Chain };
1529 SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1529 SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1530 SDValue Chain = Op.getOperand(0);
1531 SDValue Cond = Op.getOperand(1);
1532 SDValue Jump = Op.getOperand(2);
1538 SDValue R600TargetLowering::lowerFrameIndex(SDValue Op,
1538 SDValue R600TargetLowering::lowerFrameIndex(SDValue Op,
1578 SDValue R600TargetLowering::LowerFormalArguments(
1579 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1581 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1606 SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT);
1639 SDValue Arg = DAG.getLoad(
1687 static SDValue CompactSwizzlableVector(
1688 SelectionDAG &DAG, SDValue VectorEntry,
1695 SDValue NewBldVec[4];
1737 static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry,
1737 static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry,
1744 SDValue NewBldVec[4];
1777 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4],
1777 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4],
1777 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4],
1801 SDValue R600TargetLowering::constBufferLoad(LoadSDNode *LoadNode, int Block,
1805 SDValue Chain = LoadNode->getChain();
1806 SDValue Ptr = LoadNode->getBasePtr();
1818 SDValue Slots[4];
1825 SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1835 SDValue Result = DAG.getBuildVector(NewVT, DL, makeArrayRef(Slots, NumElements));
1840 SDValue MergedValues[2] = {
1851 SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
1859 SDValue Arg = N->getOperand(0);
1873 SDValue FNeg = N->getOperand(0);
1877 SDValue SelectCC = FNeg.getOperand(0);
1899 SDValue InVec = N->getOperand(0);
1900 SDValue InVal = N->getOperand(1);
1901 SDValue EltNo = N->getOperand(2);
1921 SmallVector<SDValue, 8> Ops;
1951 SDValue Arg = N->getOperand(0);
1973 if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI))
1981 SDValue LHS = N->getOperand(0);
1986 SDValue RHS = N->getOperand(1);
1987 SDValue True = N->getOperand(2);
1988 SDValue False = N->getOperand(3);
2019 SDValue Arg = N->getOperand(1);
2023 SDValue NewArgs[8] = {
2037 SDValue Arg = N->getOperand(1);
2041 SDValue NewArgs[19] = {
2068 SDValue Ptr = LoadNode->getBasePtr();
2082 SDValue &Src, SDValue &Neg, SDValue &Abs,
2082 SDValue &Src, SDValue &Neg, SDValue &Abs,
2082 SDValue &Src, SDValue &Neg, SDValue &Abs,
2083 SDValue &Sel, SDValue &Imm,
2083 SDValue &Sel, SDValue &Imm,
2109 SDValue CstOffset = Src.getOperand(0);
2220 SDValue FakeOp;
2222 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end());
2258 SDValue &Src = Ops[OperandIdx[i] - 1];
2259 SDValue &Neg = Ops[NegIdx[i] - 1];
2260 SDValue &Abs = Ops[AbsIdx[i] - 1];
2265 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2271 SDValue &Src = Ops[i];
2296 SDValue &Src = Ops[OperandIdx[i] - 1];
2297 SDValue &Neg = Ops[NegIdx[i] - 1];
2298 SDValue FakeAbs;
2299 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2307 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2308 SDValue &Imm = Ops[ImmIdx];
lib/Target/AMDGPU/R600ISelLowering.h 35 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
35 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
36 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
38 SmallVectorImpl<SDValue> &Results,
41 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
41 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
45 SmallVectorImpl<SDValue> &InVals) const override;
63 SDValue LowerImplicitParameter(SelectionDAG &DAG, EVT VT, const SDLoc &DL,
68 SDValue OptimizeSwizzle(SDValue BuildVector, SDValue Swz[], SelectionDAG &DAG,
68 SDValue OptimizeSwizzle(SDValue BuildVector, SDValue Swz[], SelectionDAG &DAG,
68 SDValue OptimizeSwizzle(SDValue BuildVector, SDValue Swz[], SelectionDAG &DAG,
70 SDValue vectorToVerticalVector(SelectionDAG &DAG, SDValue Vector) const;
70 SDValue vectorToVerticalVector(SelectionDAG &DAG, SDValue Vector) const;
72 SDValue lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const;
72 SDValue lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const;
73 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
73 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
74 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
74 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
75 SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
75 SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
77 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
77 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
79 SDValue lowerPrivateTruncStore(StoreSDNode *Store, SelectionDAG &DAG) const;
80 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
80 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
81 SDValue lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const;
81 SDValue lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
84 SDValue lowerPrivateExtLoad(SDValue Op, SelectionDAG &DAG) const;
84 SDValue lowerPrivateExtLoad(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerSHLParts(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerSHLParts(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerSRXParts(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerSRXParts(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
90 SDValue LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
93 SDValue stackPtrToRegIndex(SDValue Ptr, unsigned StackWidth,
93 SDValue stackPtrToRegIndex(SDValue Ptr, unsigned StackWidth,
97 bool isZero(SDValue Op) const;
98 bool isHWTrueValue(SDValue Op) const;
99 bool isHWFalseValue(SDValue Op) const;
101 bool FoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src,
102 SDValue &Neg, SDValue &Abs, SDValue &Sel, SDValue &Imm,
102 SDValue &Neg, SDValue &Abs, SDValue &Sel, SDValue &Imm,
102 SDValue &Neg, SDValue &Abs, SDValue &Sel, SDValue &Imm,
102 SDValue &Neg, SDValue &Abs, SDValue &Sel, SDValue &Imm,
104 SDValue constBufferLoad(LoadSDNode *LoadNode, int Block,
lib/Target/AMDGPU/SIISelLowering.cpp 1397 SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
1399 SDValue Chain,
1413 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
1419 SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
1426 SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
1427 const SDLoc &SL, SDValue Val,
1457 SDValue SITargetLowering::lowerKernargMemParameter(
1459 const SDLoc &SL, SDValue Chain,
1478 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
1479 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, 4,
1483 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
1484 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
1486 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
1494 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
1495 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Align,
1499 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
1503 SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
1504 const SDLoc &SL, SDValue Chain,
1521 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1522 SDValue ArgValue;
1552 SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
2017 SDValue SITargetLowering::LowerFormalArguments(
2018 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2020 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2112 SmallVector<SDValue, 16> Chains;
2138 SDValue Arg = lowerKernargMemParameter(
2157 SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
2171 SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
2258 SDValue
2259 SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2262 const SmallVectorImpl<SDValue> &OutVals,
2288 SDValue Flag;
2289 SmallVector<SDValue, 48> RetOps;
2295 SDValue ReturnAddrReg = CreateLiveInRegister(
2298 SDValue ReturnAddrVirtualReg = DAG.getRegister(
2313 SDValue Arg = OutVals[RealRVLocIdx];
2368 SDValue SITargetLowering::LowerCallResult(
2369 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
2369 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
2371 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
2372 SDValue ThisVal) const {
2384 SDValue Val;
2430 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
2431 SmallVectorImpl<SDValue> &MemOpChains,
2432 SDValue Chain) const {
2483 SDValue InputReg;
2498 SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2527 SDValue InputReg;
2535 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
2543 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
2564 SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2585 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
2587 const SmallVectorImpl<SDValue> &OutVals,
2668 SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
2669 SmallVectorImpl<SDValue> &InVals) const {
2673 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
2675 SDValue Chain = CLI.Chain;
2676 SDValue Callee = CLI.Callee;
2762 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2769 SmallVector<SDValue, 4> CopyFromChains;
2772 SDValue ScratchRSrcReg
2779 SmallVector<SDValue, 8> MemOpChains;
2786 SDValue Arg = OutVals[realArgIdx];
2816 SDValue DstAddr;
2822 SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
2858 SDValue SizeNode =
2860 SDValue Cpy = DAG.getMemcpy(
2869 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo,
2884 SDValue InFlag;
2892 SDValue PhysReturnAddrReg;
2898 SDValue ReturnAddrReg = CreateLiveInRegister(
2919 std::vector<SDValue> Ops;
2964 SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
3948 SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
3948 SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
3954 SDValue Lo, Hi;
3958 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
3960 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
3968 SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
3968 SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
3974 SDValue Lo0, Hi0;
3976 SDValue Lo1, Hi1;
3981 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
3983 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
3989 SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
3989 SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
3995 SDValue Lo0, Hi0;
3997 SDValue Lo1, Hi1;
3999 SDValue Lo2, Hi2;
4004 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Lo2,
4006 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Hi2,
4013 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
4013 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
4019 SDValue Result = LowerLOAD(Op, DAG);
4086 static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
4086 static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
4098 SmallVector<SDValue, 4> Elts;
4100 for (SDValue &Elt : Elts)
4113 SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode,
4116 ArrayRef<SDValue> Ops,
4133 SDValue Load
4141 SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
4146 SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
4148 ArrayRef<SDValue> Ops) const {
4174 SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
4181 static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI,
4192 SDValue LHS = N->getOperand(1);
4193 SDValue RHS = N->getOperand(2);
4210 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS,
4217 static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI,
4228 SDValue Src0 = N->getOperand(1);
4229 SDValue Src1 = N->getOperand(2);
4242 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0,
4250 SmallVectorImpl<SDValue> &Results,
4254 if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
4259 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
4267 SDValue Src0 = N->getOperand(1);
4268 SDValue Src1 = N->getOperand(2);
4270 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32,
4279 SDValue Src0 = N->getOperand(1);
4280 SDValue Src1 = N->getOperand(2);
4297 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
4306 if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
4324 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
4325 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
4334 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT,
4347 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4349 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32,
4360 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4362 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32,
4374 static SDNode *findUser(SDValue Value, unsigned Opcode) {
4434 SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND,
4434 SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND,
4439 SDValue Target = BRCOND.getOperand(2);
4476 SmallVector<SDValue, 4> Ops;
4489 SDValue Ops[] = {
4499 SDValue Ops[] = {
4503 SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops);
4508 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
4533 SDValue SITargetLowering::LowerRETURNADDR(SDValue Op,
4533 SDValue SITargetLowering::LowerRETURNADDR(SDValue Op,
4558 SDValue SITargetLowering::getFPExtOrFPTrunc(SelectionDAG &DAG,
4559 SDValue Op,
4567 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
4567 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
4571 SDValue Src = Op.getOperand(0);
4578 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src);
4579 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
4583 SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
4583 SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
4602 SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
4602 SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
4604 SDValue Chain = Op.getOperand(0);
4614 SDValue QueuePtr = CreateLiveInRegister(
4616 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
4617 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
4619 SDValue Ops[] = {
4628 SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
4628 SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
4630 SDValue Chain = Op.getOperand(0);
4644 SDValue Ops[] = {
4651 SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
4666 SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16);
4667 SDValue ApertureReg = SDValue(
4669 SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32);
4678 SDValue QueuePtr = CreateLiveInRegister(
4685 SDValue Ptr = DAG.getObjectPtrOffset(DL, QueuePtr, StructOffset);
4700 SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
4700 SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
4705 SDValue Src = ASC->getOperand(0);
4706 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
4718 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
4719 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
4720 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
4734 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
4736 SDValue NonNull
4739 SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG);
4740 SDValue CvtPtr
4764 SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
4764 SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
4766 SDValue Vec = Op.getOperand(0);
4767 SDValue Ins = Op.getOperand(1);
4768 SDValue Idx = Op.getOperand(2);
4777 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
4785 SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4785 SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4787 SDValue Vec = Op.getOperand(0);
4788 SDValue InsVal = Op.getOperand(1);
4789 SDValue Idx = Op.getOperand(2);
4803 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
4805 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
4807 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
4810 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
4811 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
4815 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16,
4822 SDValue Concat = InsertLo ?
4839 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
4843 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
4846 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
4848 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
4849 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
4853 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
4854 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT,
4857 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS);
4861 SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4861 SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4866 SDValue Vec = Op.getOperand(0);
4867 SDValue Idx = Op.getOperand(1);
4879 if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI))
4886 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
4889 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
4891 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
4892 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
4895 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
4907 SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
4907 SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
4927 SmallVector<SDValue, 4> Pieces;
4933 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL,
4945 SDValue Vec0 = SVN->getOperand(VecIdx0);
4946 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
4949 SDValue Vec1 = SVN->getOperand(VecIdx1);
4950 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
4959 SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
4959 SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
4969 SDValue Lo = DAG.getBuildVector(HalfVT, SL,
4971 SDValue Hi = DAG.getBuildVector(HalfVT, SL,
4974 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo);
4975 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi);
4977 SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi });
4984 SDValue Lo = Op.getOperand(0);
4985 SDValue Hi = Op.getOperand(1);
4990 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
4997 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
5005 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi);
5018 static SDValue
5052 SDValue PtrLo =
5054 SDValue PtrHi;
5064 SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
5065 SDValue Op,
5081 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(),
5092 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT,
5107 SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
5107 SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
5108 const SDLoc &DL, SDValue V) const {
5123 SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG,
5124 SDValue Op,
5128 SDValue Param = lowerKernargMemParameter(DAG, MVT::i32, MVT::i32, SL,
5135 static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
5144 static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
5153 static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
5154 ArrayRef<SDValue> Elts) {
5177 SmallVector<SDValue, 16> VecElts(NumElts);
5179 SDValue Elt = Elts[i];
5192 static bool parseCachePolicy(SDValue CachePolicy, SelectionDAG &DAG,
5193 SDValue *GLC, SDValue *SLC, SDValue *DLC) {
5193 SDValue *GLC, SDValue *SLC, SDValue *DLC) {
5193 SDValue *GLC, SDValue *SLC, SDValue *DLC) {
5217 static SDValue constructRetValue(SelectionDAG &DAG,
5254 SDValue N = SDValue(Result, 0);
5255 SDValue CastRes = DAG.getNode(ISD::BITCAST, DL, CastVT, N);
5258 SmallVector<SDValue, 4> BVElts;
5269 SDValue PreTFCRes;
5271 SDValue NewVec = DAG.getBuildVector(AdjVT, DL, BVElts);
5291 SmallVector<SDValue, 1> TFCElt;
5293 SDValue TFCRes = DAG.getNode(ISD::BITCAST, DL, ResultTypes[1], TFCElt[0]);
5297 static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
5297 static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
5298 SDValue *LWE, bool &IsTexFail) {
5315 SDValue SITargetLowering::lowerImage(SDValue Op,
5315 SDValue SITargetLowering::lowerImage(SDValue Op,
5335 SDValue VData;
5348 SDValue VData2 = Op.getOperand(3);
5416 SmallVector<SDValue, 4> VAddrs;
5449 SDValue AddrLo, AddrHi;
5490 SDValue VAddr;
5494 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1);
5495 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1);
5497 SDValue Unorm;
5509 SDValue TFE;
5510 SDValue LWE;
5511 SDValue TexFail = Op.getOperand(CtrlIdx);
5534 SDValue Undef = DAG.getUNDEF(Op.getValueType());
5553 SDValue GLC;
5554 SDValue SLC;
5555 SDValue DLC;
5567 SmallVector<SDValue, 26> Ops;
5571 for (const SDValue &Addr : VAddrs)
5624 SmallVector<SDValue, 1> Elt;
5638 SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
5638 SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
5639 SDValue Offset, SDValue GLC, SDValue DLC,
5639 SDValue Offset, SDValue GLC, SDValue DLC,
5639 SDValue Offset, SDValue GLC, SDValue DLC,
5649 SDValue Ops[] = {
5661 SmallVector<SDValue, 4> Loads;
5676 SDValue Ops[] = {
5704 SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
5704 SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
5770 SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
5771 SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq,
5862 SDValue GLC;
5863 SDValue DLC = DAG.getTargetConstant(0, DL, MVT::i1);
5873 SDValue ToM0 = DAG.getCopyToReg(DAG.getEntryNode(), DL, AMDGPU::M0,
5879 SDValue S = DAG.getNode(
5887 SDValue Ops[] = {
5901 SDValue Ops[] = {
5915 SDValue ToM0 = DAG.getCopyToReg(DAG.getEntryNode(), DL, AMDGPU::M0,
5917 SDValue Ops[] = {
5978 SDValue Numerator = Op.getOperand(1);
5979 SDValue Denominator = Op.getOperand(2);
5986 SDValue Src0 = Param->isAllOnesValue() ? Numerator : Denominator;
6043 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32,
6063 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0,
6072 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
6073 SDValue SrcVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32,
6076 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec,
6094 static unsigned getBufferOffsetForMMO(SDValue VOffset,
6095 SDValue SOffset,
6096 SDValue Offset,
6097 SDValue VIndex = SDValue()) {
6113 SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
6113 SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
6122 SDValue Chain = M->getOperand(0);
6123 SDValue M0 = M->getOperand(2);
6124 SDValue Value = M->getOperand(3);
6187 SDValue Ops[] = {
6232 SDValue Ops[] = {
6248 SDValue Ops[] = {
6290 SDValue Ops[] = {
6310 SDValue Ops[] = {
6337 SDValue Ops[] = {
6361 SDValue Ops[] = {
6385 SDValue Ops[] = {
6418 SDValue Ops[] = {
6490 SDValue Ops[] = {
6564 SDValue Ops[] = {
6631 SDValue Ops[] = {
6656 SDValue Ops[] = {
6677 SDValue Ops[] = {
6709 SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
6711 ArrayRef<SDValue> Ops, EVT MemVT,
6729 auto NewOp = DAG.getMemIntrinsicNode(Opcode, DL, WidenedVTList, Ops,
6732 auto Extract = DAG.getNode(
6740 SDValue SITargetLowering::handleD16VData(SDValue VData,
6740 SDValue SITargetLowering::handleD16VData(SDValue VData,
6754 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData);
6758 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, EquivStoreVT, IntVData);
6766 SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
6766 SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
6769 SDValue Chain = Op.getOperand(0);
6780 const SDValue Ops[] = {
6799 SDValue Src0 = Op.getOperand(4);
6800 SDValue Src1 = Op.getOperand(5);
6804 SDValue Undef = DAG.getUNDEF(MVT::f32);
6805 const SDValue Ops[] = {
6832 SDValue VData = Op.getOperand(2);
6843 SDValue Ops[] = {
6863 SDValue VData = Op.getOperand(2);
6868 SDValue Ops[] = {
6888 SDValue VData = Op.getOperand(2);
6893 SDValue Ops[] = {
6914 SDValue VData = Op.getOperand(2);
6923 SDValue Ops[] = {
6958 SDValue VData = Op.getOperand(2);
6972 SDValue Ops[] = {
7002 SDValue VData = Op.getOperand(2);
7017 SDValue Ops[] = {
7049 SDValue Ops[] = {
7076 SDValue Ops[] = {
7111 std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
7111 std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
7112 SDValue Offset, SelectionDAG &DAG) const {
7115 SDValue N0 = Offset;
7142 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
7146 SDValue Ops[] = { N0, OverflowVal };
7161 unsigned SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
7162 SelectionDAG &DAG, SDValue *Offsets,
7176 SDValue N0 = CombinedOffset.getOperand(0);
7177 SDValue N1 = CombinedOffset.getOperand(1);
7195 SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
7197 ArrayRef<SDValue> Ops,
7204 SDValue BufferLoad = DAG.getMemIntrinsicNode(Opc, DL, ResList,
7207 SDValue LoadVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, BufferLoad);
7214 SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
7216 SDValue Ops[],
7221 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]);
7225 ArrayRef<SDValue> OpsRef = makeArrayRef(&Ops[0], 9);
7230 static SDValue getLoadExtOrTrunc(SelectionDAG &DAG,
7231 ISD::LoadExtType ExtType, SDValue Op,
7250 SDValue SITargetLowering::widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const {
7276 SDValue Ptr = Ld->getBasePtr();
7277 SDValue NewLoad = DAG.getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD,
7293 SDValue Cvt = NewLoad;
7320 SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
7320 SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
7333 SDValue Chain = Load->getChain();
7334 SDValue BasePtr = Load->getBasePtr();
7339 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain,
7343 SDValue Ops[] = {
7351 SmallVector<SDValue, 3> Elts;
7353 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD,
7359 SDValue Ops[] = {
7375 SDValue Ops[2];
7487 SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
7487 SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
7492 SDValue Cond = Op.getOperand(0);
7494 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
7495 SDValue One = DAG.getConstant(1, DL, MVT::i32);
7497 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1));
7498 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2));
7500 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero);
7501 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero);
7503 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1);
7505 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One);
7506 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One);
7508 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1);
7510 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi});
7516 SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
7516 SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
7519 SDValue LHS = Op.getOperand(0);
7520 SDValue RHS = Op.getOperand(1);
7552 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
7561 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
7568 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
7569 EVT VT, SDValue A, SDValue B, SDValue GlueChain) {
7569 EVT VT, SDValue A, SDValue B, SDValue GlueChain) {
7569 EVT VT, SDValue A, SDValue B, SDValue GlueChain) {
7588 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
7589 EVT VT, SDValue A, SDValue B, SDValue C,
7589 EVT VT, SDValue A, SDValue B, SDValue C,
7589 EVT VT, SDValue A, SDValue B, SDValue C,
7590 SDValue GlueChain) {
7609 SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
7609 SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
7610 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
7614 SDValue Src0 = Op.getOperand(0);
7615 SDValue Src1 = Op.getOperand(1);
7617 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
7618 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
7620 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1);
7621 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1);
7623 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32);
7624 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag);
7630 SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
7630 SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
7632 SDValue LHS = Op.getOperand(1);
7633 SDValue RHS = Op.getOperand(2);
7635 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS);
7638 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32);
7641 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32);
7643 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
7648 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT);
7650 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One);
7656 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1);
7658 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0);
7665 static const SDValue getSPDenormModeValue(int SPDenormMode, SelectionDAG &DAG,
7676 SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
7676 SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
7677 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
7681 SDValue LHS = Op.getOperand(0);
7682 SDValue RHS = Op.getOperand(1);
7684 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
7688 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT,
7690 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT,
7694 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32,
7696 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32,
7702 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i16);
7707 SDValue EnableDenorm;
7709 const SDValue EnableDenormValue =
7715 const SDValue EnableDenormValue = DAG.getConstant(FP_DENORM_FLUSH_NONE,
7722 SDValue Ops[3] = {
7731 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0,
7734 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp,
7737 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled,
7740 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul,
7743 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma2, Fma1, Mul, Fma2);
7745 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3,
7750 SDValue DisableDenorm;
7752 const SDValue DisableDenormValue =
7759 const SDValue DisableDenormValue =
7767 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
7772 SDValue Scale = NumeratorScaled.getValue(1);
7773 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32,
7779 SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
7779 SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
7784 SDValue X = Op.getOperand(0);
7785 SDValue Y = Op.getOperand(1);
7787 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
7791 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X);
7793 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0);
7795 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0);
7797 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One);
7799 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp);
7801 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One);
7803 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X);
7805 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1);
7806 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3);
7808 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64,
7811 SDValue Scale;
7817 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
7820 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
7821 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y);
7822 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0);
7823 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1);
7825 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi);
7826 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi);
7828 SDValue Scale0Hi
7830 SDValue Scale1Hi
7833 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ);
7834 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ);
7840 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64,
7846 SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
7846 SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
7861 SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
7861 SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
7945 SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
7945 SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
7948 SDValue Arg = Op.getOperand(0);
7949 SDValue TrigVal;
7953 SDValue OneOver2Pi = DAG.getConstantFP(0.5 / M_PI, DL, VT);
7956 SDValue MulVal = DAG.getNode(ISD::FMUL, DL, VT, Arg, OneOver2Pi);
7972 SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
7972 SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
7984 SDValue ChainIn = Op.getOperand(0);
7985 SDValue Addr = Op.getOperand(1);
7986 SDValue Old = Op.getOperand(2);
7987 SDValue New = Op.getOperand(3);
7992 SDValue NewOld = DAG.getBuildVector(VecType, DL, {New, Old});
7993 SDValue Ops[] = { ChainIn, Addr, NewOld };
8003 SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N,
8013 SDValue Src = N->getOperand(0);
8022 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Src);
8045 SDValue SITargetLowering::performSHLPtrCombine(SDNode *N,
8049 SDValue N0 = N->getOperand(0);
8050 SDValue N1 = N->getOperand(1);
8081 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1);
8082 SDValue COffset = DAG.getConstant(Offset, SL, MVT::i32);
8092 SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
8094 SDValue Ptr = N->getBasePtr();
8100 SDValue NewPtr = performSHLPtrCombine(Ptr.getNode(), N->getAddressSpace(),
8103 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
8124 SDValue SITargetLowering::splitBinaryBitConstantOp(
8127 unsigned Opc, SDValue LHS,
8148 static bool isBoolSGPR(SDValue V) {
8185 static uint32_t getPermuteMask(SelectionDAG &DAG, SDValue V) {
8228 SDValue SITargetLowering::performAndCombine(SDNode *N,
8235 SDValue LHS = N->getOperand(0);
8236 SDValue RHS = N->getOperand(1);
8241 if (SDValue Split
8261 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
8266 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE,
8268 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(LHS), VT, Ext,
8296 SDValue X = LHS.getOperand(0);
8297 SDValue Y = RHS.getOperand(0);
8414 SDValue SITargetLowering::performOrCombine(SDNode *N,
8417 SDValue LHS = N->getOperand(0);
8418 SDValue RHS = N->getOperand(1);
8425 SDValue Src = LHS.getOperand(0);
8513 SDValue ExtSrc = RHS.getOperand(0);
8517 SDValue LowLHS, HiBits;
8519 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc);
8524 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
8532 if (SDValue Split
8540 SDValue SITargetLowering::performXorCombine(SDNode *N,
8546 SDValue LHS = N->getOperand(0);
8547 SDValue RHS = N->getOperand(1);
8551 if (SDValue Split
8616 SDValue SITargetLowering::performZeroExtendCombine(SDNode *N,
8626 SDValue Src = N->getOperand(0);
8633 SDValue BCSrc = Src.getOperand(0);
8642 SDValue SITargetLowering::performSignExtendInRegCombine(SDNode *N,
8645 SDValue Src = N->getOperand(0);
8654 SDValue Ops[] = {
8669 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(Opc, SDLoc(N),
8679 SDValue SITargetLowering::performClassCombine(SDNode *N,
8682 SDValue Mask = N->getOperand(1);
8696 SDValue SITargetLowering::performRcpCombine(SDNode *N,
8699 SDValue N0 = N->getOperand(0);
8713 bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
8813 SDValue SrcOp = Op.getOperand(i);
8834 SDValue Src = Op.getOperand(0);
8837 SDValue TruncSrc = Src.getOperand(0);
8872 SDValue SITargetLowering::getCanonicalConstantFP(
8898 static bool vectorEltWillFoldAway(SDValue Op) {
8902 SDValue SITargetLowering::performFCanonicalizeCombine(
8906 SDValue N0 = N->getOperand(0);
8930 SDValue NewElts[2];
8931 SDValue Lo = N0.getOperand(0);
8932 SDValue Hi = N0.getOperand(1);
8937 SDValue Op = N0.getOperand(I);
8978 SDValue Canon0 = DAG.getNode(ISD::FCANONICALIZE, SL, VT,
8980 SDValue Canon1 = getCanonicalConstantFP(DAG, SL, VT, CRHS->getValueAPF());
9011 SDValue SITargetLowering::performIntMed3ImmCombine(
9013 SDValue Op0, SDValue Op1, bool Signed) const {
9013 SDValue Op0, SDValue Op1, bool Signed) const {
9041 SDValue Tmp1 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(0));
9042 SDValue Tmp2 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(1));
9043 SDValue Tmp3 = DAG.getNode(ExtOp, SL, NVT, Op1);
9045 SDValue Med3 = DAG.getNode(Med3Opc, SL, NVT, Tmp1, Tmp2, Tmp3);
9049 static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
9061 SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
9063 SDValue Op0,
9064 SDValue Op1) const {
9097 SDValue Var = Op0.getOperand(0);
9115 SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
9121 SDValue Op0 = N->getOperand(0);
9122 SDValue Op1 = N->getOperand(1);
9159 if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, true))
9164 if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, false))
9177 if (SDValue Res = performFPMed3ImmCombine(DAG, SDLoc(N), Op0, Op1))
9184 static bool isClampZeroToOne(SDValue A, SDValue B) {
9184 static bool isClampZeroToOne(SDValue A, SDValue B) {
9197 SDValue SITargetLowering::performFMed3Combine(SDNode *N,
9206 SDValue Src0 = N->getOperand(0);
9207 SDValue Src1 = N->getOperand(1);
9208 SDValue Src2 = N->getOperand(2);
9241 SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
9243 SDValue Src0 = N->getOperand(0);
9244 SDValue Src1 = N->getOperand(1);
9250 SDValue SITargetLowering::performExtractVectorEltCombine(
9252 SDValue Vec = N->getOperand(0);
9262 SDValue Idx = N->getOperand(1);
9263 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9276 SDValue Idx = N->getOperand(1);
9295 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9297 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9318 SDValue Idx = N->getOperand(1);
9320 SDValue V;
9322 SDValue IC = DAG.getConstant(I, SL, IdxVT);
9323 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
9352 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
9355 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
9358 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
9362 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, EltVT.changeTypeToInteger(), Srl);
9370 SDValue
9373 SDValue Vec = N->getOperand(0);
9374 SDValue Idx = N->getOperand(2);
9392 SDValue Ins = N->getOperand(1);
9395 SmallVector<SDValue, 16> Ops;
9397 SDValue IC = DAG.getConstant(I, SL, IdxVT);
9398 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
9399 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ);
9432 SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
9439 SDValue Op0 = N->getOperand(0);
9440 SDValue Op1 = N->getOperand(1);
9451 SDValue Op2 = Op1.getOperand(1);
9466 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1);
9470 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL,
9472 SDValue N0, SDValue N1, SDValue N2,
9472 SDValue N0, SDValue N1, SDValue N2,
9472 SDValue N0, SDValue N1, SDValue N2,
9476 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2);
9480 SDValue SITargetLowering::performAddCombine(SDNode *N,
9485 SDValue LHS = N->getOperand(0);
9486 SDValue RHS = N->getOperand(1);
9495 SDValue MulLHS = LHS.getOperand(0);
9496 SDValue MulRHS = LHS.getOperand(1);
9497 SDValue AddRHS = RHS;
9518 if (SDValue V = reassociateScalarOps(N, DAG)) {
9538 auto Cond = RHS.getOperand(0);
9542 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
9550 SDValue Args[] = { LHS, RHS.getOperand(0), RHS.getOperand(2) };
9557 SDValue SITargetLowering::performSubCombine(SDNode *N,
9566 SDValue LHS = N->getOperand(0);
9567 SDValue RHS = N->getOperand(1);
9574 SDValue Args[] = { LHS.getOperand(0), RHS, LHS.getOperand(2) };
9580 SDValue SITargetLowering::performAddCarrySubCarryCombine(SDNode *N,
9591 SDValue LHS = N->getOperand(0);
9599 SDValue Args[] = { LHS.getOperand(0), LHS.getOperand(1), N->getOperand(2) };
9605 SDValue SITargetLowering::performFAddCombine(SDNode *N,
9614 SDValue LHS = N->getOperand(0);
9615 SDValue RHS = N->getOperand(1);
9622 SDValue A = LHS.getOperand(0);
9626 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
9634 SDValue A = RHS.getOperand(0);
9638 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
9647 SDValue SITargetLowering::performFSubCombine(SDNode *N,
9662 SDValue LHS = N->getOperand(0);
9663 SDValue RHS = N->getOperand(1);
9666 SDValue A = LHS.getOperand(0);
9670 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
9671 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
9681 SDValue A = RHS.getOperand(0);
9685 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
9694 SDValue SITargetLowering::performFMACombine(SDNode *N,
9705 SDValue Op1 = N->getOperand(0);
9706 SDValue Op2 = N->getOperand(1);
9707 SDValue FMA = N->getOperand(2);
9727 SDValue Vec1 = Op1.getOperand(0);
9728 SDValue Idx1 = Op1.getOperand(1);
9729 SDValue Vec2 = Op2.getOperand(0);
9731 SDValue FMAOp1 = FMA.getOperand(0);
9732 SDValue FMAOp2 = FMA.getOperand(1);
9733 SDValue FMAAcc = FMA.getOperand(2);
9745 SDValue Vec3 = FMAOp1.getOperand(0);
9746 SDValue Vec4 = FMAOp2.getOperand(0);
9747 SDValue Idx2 = FMAOp1.getOperand(1);
9769 SDValue SITargetLowering::performSetCCCombine(SDNode *N,
9774 SDValue LHS = N->getOperand(0);
9775 SDValue RHS = N->getOperand(1);
9865 SDValue SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
9871 SDValue Src = N->getOperand(0);
9872 SDValue Srl = N->getOperand(0);
9908 SDValue SITargetLowering::performClampCombine(SDNode *N,
9933 SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
9967 if (SDValue Widended = widenLoad(cast<LoadSDNode>(N), DCI))
10018 SDValue Src = N->getOperand(0);
10044 SDValue Src = N->getOperand(0);
10049 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src);
10179 SmallVector<SDValue, 12> Ops;
10220 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
10237 static bool isFrameIndexOp(SDValue Op) {
10251 SDValue SrcVal = Node->getOperand(2);
10259 SDValue VReg = DAG.getRegister(
10263 SDValue ToVReg
10266 SDValue ToResultReg
10275 SmallVector<SDValue, 8> Ops;
10315 SDValue Src0 = Node->getOperand(0);
10316 SDValue Src1 = Node->getOperand(1);
10317 SDValue Src2 = Node->getOperand(2);
10329 SDValue UndefReg = DAG.getRegister(MRI.createVirtualRegister(RC), VT);
10331 SDValue ImpDef = DAG.getCopyToReg(DAG.getEntryNode(), SDLoc(Node),
10352 SmallVector<SDValue, 4> Ops = { Src0, Src1, Src2 };
10365 SDValue VDstIn = Node->getOperand(6);
10371 SmallVector<SDValue, 8> Ops = { SDValue(FI, 0), Node->getOperand(1),
10464 static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
10466 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32);
10472 SDValue Ptr) const {
10478 const SDValue Ops0[] = {
10486 SDValue SubRegHi = SDValue(DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL,
10490 const SDValue Ops1[] = {
10506 SDValue Ptr, uint32_t RsrcDword1,
10508 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr);
10509 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr);
10516 SDValue DataLo = buildSMovImm32(DAG, DL,
10518 SDValue DataHi = buildSMovImm32(DAG, DL, RsrcDword2And3 >> 32);
10520 const SDValue Ops[] = {
10727 void SITargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
10893 bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
lib/Target/AMDGPU/SIISelLowering.h 40 SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL,
41 SDValue Chain, uint64_t Offset) const;
42 SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const;
43 SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
44 const SDLoc &SL, SDValue Chain,
48 SDValue lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
49 const SDLoc &SL, SDValue Chain,
51 SDValue getPreloadedValue(SelectionDAG &DAG,
56 SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
56 SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
58 SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
58 SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
60 SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
60 SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
62 SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
62 SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
62 SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
63 SDValue GLC, SDValue DLC, SelectionDAG &DAG) const;
63 SDValue GLC, SDValue DLC, SelectionDAG &DAG) const;
65 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
65 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
66 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
66 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
67 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
67 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
75 std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
75 std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
75 std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
78 SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
79 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
79 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
80 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
80 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
81 SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
81 SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const;
83 SDValue LowerFDIV16(SDValue Op, SelectionDAG &DAG) const;
83 SDValue LowerFDIV16(SDValue Op, SelectionDAG &DAG) const;
84 SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
84 SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, bool Signed) const;
87 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, bool Signed) const;
88 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
93 SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M,
94 SelectionDAG &DAG, ArrayRef<SDValue> Ops,
97 SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
98 ArrayRef<SDValue> Ops) const;
102 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
103 ArrayRef<SDValue> Ops, EVT MemVT,
106 SDValue handleD16VData(SDValue VData, SelectionDAG &DAG) const;
106 SDValue handleD16VData(SDValue VData, SelectionDAG &DAG) const;
110 SDValue getFPExtOrFPTrunc(SelectionDAG &DAG,
111 SDValue Op,
115 SDValue convertArgType(
116 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Val,
120 SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
120 SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
121 SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
121 SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
123 SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
126 SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
126 SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
127 SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
127 SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
128 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
128 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
129 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
129 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
130 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
130 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
131 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
131 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
132 SDValue lowerTRAP(SDValue Op, SelectionDAG &DAG) const;
132 SDValue lowerTRAP(SDValue Op, SelectionDAG &DAG) const;
133 SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const;
133 SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const;
137 SDValue performUCharToFloatCombine(SDNode *N,
139 SDValue performSHLPtrCombine(SDNode *N,
144 SDValue performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const;
146 SDValue splitBinaryBitConstantOp(DAGCombinerInfo &DCI, const SDLoc &SL,
147 unsigned Opc, SDValue LHS,
150 SDValue performAndCombine(SDNode *N, DAGCombinerInfo &DCI) const;
151 SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const;
152 SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
153 SDValue performZeroExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const;
154 SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
155 SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const;
156 SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
158 SDValue performFCanonicalizeCombine(SDNode *N, DAGCombinerInfo &DCI) const;
160 SDValue performFPMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL,
161 SDValue Op0, SDValue Op1) const;
161 SDValue Op0, SDValue Op1) const;
162 SDValue performIntMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL,
163 SDValue Op0, SDValue Op1, bool Signed) const;
163 SDValue Op0, SDValue Op1, bool Signed) const;
164 SDValue performMinMaxCombine(SDNode *N, DAGCombinerInfo &DCI) const;
165 SDValue performFMed3Combine(SDNode *N, DAGCombinerInfo &DCI) const;
166 SDValue performCvtPkRTZCombine(SDNode *N, DAGCombinerInfo &DCI) const;
167 SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
168 SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
170 SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const;
173 SDValue performAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
174 SDValue performAddCarrySubCarryCombine(SDNode *N, DAGCombinerInfo &DCI) const;
175 SDValue performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
176 SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
177 SDValue performFSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
178 SDValue performFMACombine(SDNode *N, DAGCombinerInfo &DCI) const;
179 SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
180 SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const;
181 SDValue performClampCombine(SDNode *N, DAGCombinerInfo &DCI) const;
182 SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
208 unsigned setBufferOffsets(SDValue CombinedOffset, SelectionDAG &DAG,
209 SDValue *Offsets, unsigned Align = 4) const;
212 SDValue handleByteShortBufferLoads(SelectionDAG &DAG, EVT LoadVT, SDLoc DL,
213 ArrayRef<SDValue> Ops, MemSDNode *M) const;
216 SDValue handleByteShortBufferStores(SelectionDAG &DAG, EVT VDataType,
217 SDLoc DL, SDValue Ops[],
290 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
290 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
294 SmallVectorImpl<SDValue> &InVals) const override;
301 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
301 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
303 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
310 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
311 SmallVectorImpl<SDValue> &MemOpChains,
312 SDValue Chain) const;
314 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
314 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
314 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
318 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
319 SDValue ThisVal) const;
324 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
326 const SmallVectorImpl<SDValue> &OutVals,
329 SDValue LowerCall(CallLoweringInfo &CLI,
330 SmallVectorImpl<SDValue> &InVals) const override;
352 SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
352 SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
353 SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
353 SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
354 SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
354 SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
355 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
355 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
357 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
360 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
368 SDValue Ptr) const;
369 MachineSDNode *buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr,
375 SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
375 SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
376 SDValue V) const;
380 void computeKnownBitsForFrameIndex(const SDValue Op,
389 bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
393 bool isKnownNeverNaNForTargetNode(SDValue Op,
lib/Target/AMDGPU/SIInstrInfo.cpp 232 SDValue Off0 = Load0->getOperand(OffIdx0);
233 SDValue Off1 = Load1->getOperand(OffIdx1);
lib/Target/ARC/ARCISelDAGToDAG.cpp 47 bool SelectFrameADDR_ri(SDValue Addr, SDValue &Base, SDValue &Offset);
47 bool SelectFrameADDR_ri(SDValue Addr, SDValue &Base, SDValue &Offset);
47 bool SelectFrameADDR_ri(SDValue Addr, SDValue &Base, SDValue &Offset);
48 bool SelectAddrModeS9(SDValue Addr, SDValue &Base, SDValue &Offset);
48 bool SelectAddrModeS9(SDValue Addr, SDValue &Base, SDValue &Offset);
48 bool SelectAddrModeS9(SDValue Addr, SDValue &Base, SDValue &Offset);
49 bool SelectAddrModeImm(SDValue Addr, SDValue &Base, SDValue &Offset);
49 bool SelectAddrModeImm(SDValue Addr, SDValue &Base, SDValue &Offset);
49 bool SelectAddrModeImm(SDValue Addr, SDValue &Base, SDValue &Offset);
50 bool SelectAddrModeFar(SDValue Addr, SDValue &Base, SDValue &Offset);
50 bool SelectAddrModeFar(SDValue Addr, SDValue &Base, SDValue &Offset);
50 bool SelectAddrModeFar(SDValue Addr, SDValue &Base, SDValue &Offset);
51 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
51 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
51 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
75 bool ARCDAGToDAGISel::SelectAddrModeImm(SDValue Addr, SDValue &Base,
75 bool ARCDAGToDAGISel::SelectAddrModeImm(SDValue Addr, SDValue &Base,
76 SDValue &Offset) {
85 bool ARCDAGToDAGISel::SelectAddrModeS9(SDValue Addr, SDValue &Base,
85 bool ARCDAGToDAGISel::SelectAddrModeS9(SDValue Addr, SDValue &Base,
86 SDValue &Offset) {
127 bool ARCDAGToDAGISel::SelectAddrModeFar(SDValue Addr, SDValue &Base,
127 bool ARCDAGToDAGISel::SelectAddrModeFar(SDValue Addr, SDValue &Base,
128 SDValue &Offset) {
146 bool ARCDAGToDAGISel::SelectFrameADDR_ri(SDValue Addr, SDValue &Base,
146 bool ARCDAGToDAGISel::SelectFrameADDR_ri(SDValue Addr, SDValue &Base,
147 SDValue &Offset) {
lib/Target/ARC/ARCISelLowering.cpp 36 static SDValue lowerCallResult(SDValue Chain, SDValue InFlag,
36 static SDValue lowerCallResult(SDValue Chain, SDValue InFlag,
36 static SDValue lowerCallResult(SDValue Chain, SDValue InFlag,
39 SmallVectorImpl<SDValue> &InVals);
163 SDValue ARCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
163 SDValue ARCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
164 SDValue LHS = Op.getOperand(0);
165 SDValue RHS = Op.getOperand(1);
167 SDValue TVal = Op.getOperand(2);
168 SDValue FVal = Op.getOperand(3);
172 SDValue Cmp = DAG.getNode(ARCISD::CMP, dl, MVT::Glue, LHS, RHS);
177 SDValue ARCTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
177 SDValue ARCTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
179 SDValue Op0 = Op.getOperand(0);
190 SDValue LS = DAG.getNode(ISD::SHL, dl, MVT::i32, Op0,
192 SDValue SR = DAG.getNode(ISD::SRA, dl, MVT::i32, LS,
197 SDValue ARCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
197 SDValue ARCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
198 SDValue Chain = Op.getOperand(0);
200 SDValue LHS = Op.getOperand(2);
201 SDValue RHS = Op.getOperand(3);
202 SDValue Dest = Op.getOperand(4);
210 SDValue ARCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
210 SDValue ARCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
212 SDValue GA = DAG.getTargetJumpTable(N->getIndex(), MVT::i32);
223 SDValue ARCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
224 SmallVectorImpl<SDValue> &InVals) const {
228 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
230 SDValue Chain = CLI.Chain;
231 SDValue Callee = CLI.Callee;
257 SmallVector<std::pair<unsigned, SDValue>, 4> RegsToPass;
258 SmallVector<SDValue, 12> MemOpChains;
260 SDValue StackPtr;
264 SDValue Arg = OutVals[i];
293 SDValue SOffset = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);
294 SDValue PtrOff = DAG.getNode(
297 SDValue Store =
313 SDValue Glue;
335 SmallVector<SDValue, 8> Ops;
370 static SDValue lowerCallResult(SDValue Chain, SDValue Glue,
370 static SDValue lowerCallResult(SDValue Chain, SDValue Glue,
370 static SDValue lowerCallResult(SDValue Chain, SDValue Glue,
373 SmallVectorImpl<SDValue> &InVals) {
379 SDValue RetValue;
396 SmallVector<SDValue, 4> MemOpChains;
400 SDValue StackPtr = DAG.getRegister(ARC::SP, MVT::i32);
401 SDValue SpLoc = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr,
403 SDValue Load =
424 SDValue SDV;
431 SDValue ARCTargetLowering::LowerFormalArguments(
432 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
434 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
446 SDValue ARCTargetLowering::LowerCallArguments(
447 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
449 SmallVectorImpl<SDValue> &InVals) const {
474 SmallVector<SDValue, 4> CFRegNode;
476 SmallVector<SDValue, 4> MemOps;
481 SDValue ArgIn;
510 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
535 SDValue FIN = DAG.getFrameIndex(VarFI, MVT::i32);
540 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
542 SDValue VAObj = DAG.getNode(ISD::ADD, dl, MVT::i32, FIN,
545 SDValue Store =
569 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
604 SDValue
605 ARCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
608 const SmallVectorImpl<SDValue> &OutVals,
627 SDValue Flag;
628 SmallVector<SDValue, 4> RetOps(1, Chain);
629 SmallVector<SDValue, 4> MemOpChains;
647 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
686 SDValue ARCTargetLowering::PerformDAGCombine(SDNode *N,
709 SDValue ARCTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
709 SDValue ARCTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
723 SDValue ARCTargetLowering::LowerGlobalAddress(SDValue Op,
723 SDValue ARCTargetLowering::LowerGlobalAddress(SDValue Op,
729 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, MVT::i32, Offset);
733 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
733 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
741 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
747 SDValue ARCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
747 SDValue ARCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
lib/Target/ARC/ARCISelLowering.h 66 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
66 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
81 SDValue LowerCallArguments(SDValue Chain, CallingConv::ID CallConv,
81 SDValue LowerCallArguments(SDValue Chain, CallingConv::ID CallConv,
85 SmallVectorImpl<SDValue> &InVals) const;
87 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
93 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
95 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
95 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
99 SmallVectorImpl<SDValue> &InVals) const override;
101 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
102 SmallVectorImpl<SDValue> &InVals) const override;
104 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
104 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
106 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
lib/Target/ARM/ARMISelDAGToDAG.cpp 74 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
81 bool isShifterOpProfitable(const SDValue &Shift,
83 bool SelectRegShifterOperand(SDValue N, SDValue &A,
83 bool SelectRegShifterOperand(SDValue N, SDValue &A,
84 SDValue &B, SDValue &C,
84 SDValue &B, SDValue &C,
86 bool SelectImmShifterOperand(SDValue N, SDValue &A,
86 bool SelectImmShifterOperand(SDValue N, SDValue &A,
87 SDValue &B, bool CheckProfitability = true);
88 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
88 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
89 SDValue &B, SDValue &C) {
89 SDValue &B, SDValue &C) {
93 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
93 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
94 SDValue &B) {
99 bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
99 bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
101 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
101 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
101 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
102 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
102 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
102 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
102 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
104 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
104 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
104 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
111 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
112 SDValue &Offset, SDValue &Opc);
112 SDValue &Offset, SDValue &Opc);
113 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
114 SDValue &Offset, SDValue &Opc);
114 SDValue &Offset, SDValue &Opc);
115 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
116 SDValue &Offset, SDValue &Opc);
116 SDValue &Offset, SDValue &Opc);
117 bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
117 bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
118 bool SelectAddrMode3(SDValue N, SDValue &Base,
118 bool SelectAddrMode3(SDValue N, SDValue &Base,
119 SDValue &Offset, SDValue &Opc);
119 SDValue &Offset, SDValue &Opc);
120 bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
121 SDValue &Offset, SDValue &Opc);
121 SDValue &Offset, SDValue &Opc);
122 bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
122 bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
122 bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
123 bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
123 bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
123 bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
124 bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
124 bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
124 bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
125 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
125 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
125 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
126 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
126 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
128 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
128 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
128 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
131 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
131 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
131 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
132 bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
132 bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
132 bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
133 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
133 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
134 SDValue &OffImm);
135 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
135 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
136 SDValue &OffImm);
137 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
137 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
138 SDValue &OffImm);
139 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
139 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
140 SDValue &OffImm);
141 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
141 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
141 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
143 bool SelectTAddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
143 bool SelectTAddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
143 bool SelectTAddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
146 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
146 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
146 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
147 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
147 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
148 SDValue &OffImm);
149 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
150 SDValue &OffImm);
152 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm);
152 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm);
153 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm,
153 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm,
156 bool SelectT2AddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
156 bool SelectT2AddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
156 bool SelectT2AddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
157 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
157 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
158 SDValue &OffReg, SDValue &ShImm);
158 SDValue &OffReg, SDValue &ShImm);
159 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
159 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
159 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
215 SDValue PredicateMask);
218 SDValue PredicateMask, SDValue Inactive);
218 SDValue PredicateMask, SDValue Inactive);
268 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
269 std::vector<SDValue> &OutOps) override;
272 SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
272 SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
273 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
273 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
274 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
274 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
275 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
275 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
278 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
278 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
278 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
278 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
279 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
279 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
279 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
279 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
280 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
280 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
280 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
280 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
283 SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
283 SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
291 bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
292 unsigned &PowerOfTwo, SDValue &NewMulConst) const;
296 void replaceDAGValue(const SDValue &N, SDValue M);
296 void replaceDAGValue(const SDValue &N, SDValue M);
312 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
328 static bool isScaledConstantInRange(SDValue Node, int Scale,
367 SDValue N0 = N->getOperand(0);
368 SDValue N1 = N->getOperand(1);
395 SDValue Srl = N1.getOperand(0);
403 SDValue CPTmp0;
404 SDValue CPTmp1;
405 SDValue CPTmp2;
474 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
486 bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
489 SDValue &NewMulConst) const {
520 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
520 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
525 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
526 SDValue &BaseReg,
527 SDValue &Opc,
536 SDValue NewMulConst;
564 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
565 SDValue &BaseReg,
566 SDValue &ShReg,
567 SDValue &Opc,
593 bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
593 bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
600 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
601 SDValue &Base,
602 SDValue &OffImm) {
653 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
653 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
653 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
654 SDValue &Opc) {
746 SDValue NewMulConst;
761 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
762 SDValue &Offset, SDValue &Opc) {
762 SDValue &Offset, SDValue &Opc) {
797 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
798 SDValue &Offset, SDValue &Opc) {
798 SDValue &Offset, SDValue &Opc) {
817 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
818 SDValue &Offset, SDValue &Opc) {
818 SDValue &Offset, SDValue &Opc) {
837 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
837 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
842 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
843 SDValue &Base, SDValue &Offset,
843 SDValue &Base, SDValue &Offset,
844 SDValue &Opc) {
896 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
897 SDValue &Offset, SDValue &Opc) {
897 SDValue &Offset, SDValue &Opc) {
918 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
918 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
918 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
977 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
978 SDValue &Base, SDValue &Offset) {
978 SDValue &Base, SDValue &Offset) {
982 bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
983 SDValue &Base, SDValue &Offset) {
983 SDValue &Base, SDValue &Offset) {
987 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
987 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
988 SDValue &Align) {
1016 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1017 SDValue &Offset) {
1030 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1031 SDValue &Offset, SDValue &Label) {
1031 SDValue &Offset, SDValue &Label) {
1034 SDValue N1 = N.getOperand(1);
1048 static bool shouldUseZeroOffsetLdSt(SDValue N) {
1062 bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1062 bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1063 SDValue &Offset) {
1078 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
1078 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
1079 SDValue &Offset) {
1086 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1087 SDValue &Base, SDValue &OffImm) {
1087 SDValue &Base, SDValue &OffImm) {
1124 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1124 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1125 SDValue &OffImm) {
1130 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1130 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1131 SDValue &OffImm) {
1136 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1136 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1137 SDValue &OffImm) {
1141 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1142 SDValue &Base, SDValue &OffImm) {
1142 SDValue &Base, SDValue &OffImm) {
1188 bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base,
1188 bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base,
1189 SDValue &OffImm) {
1215 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1216 SDValue &Base, SDValue &OffImm) {
1216 SDValue &Base, SDValue &OffImm) {
1271 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1272 SDValue &Base, SDValue &OffImm) {
1272 SDValue &Base, SDValue &OffImm) {
1298 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1299 SDValue &OffImm){
1316 bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N, SDValue &Base,
1316 bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N, SDValue &Base,
1317 SDValue &OffImm) {
1344 bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
1345 SDValue &OffImm) {
1349 bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
1350 SDValue &OffImm,
1368 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1369 SDValue &Base,
1370 SDValue &OffReg, SDValue &ShImm) {
1370 SDValue &OffReg, SDValue &ShImm) {
1414 SDValue NewMulConst;
1428 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1428 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1429 SDValue &OffImm) {
1460 static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1476 SDValue Offset, AMOpc;
1523 SDValue Chain = LD->getChain();
1524 SDValue Base = LD->getBasePtr();
1525 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1533 SDValue Chain = LD->getChain();
1534 SDValue Base = LD->getBasePtr();
1535 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1564 SDValue Chain = LD->getChain();
1565 SDValue Base = LD->getBasePtr();
1566 SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1583 SDValue Offset;
1612 SDValue Chain = LD->getChain();
1613 SDValue Base = LD->getBasePtr();
1614 SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1635 SDValue Offset;
1673 SDValue Chain = LD->getChain();
1674 SDValue Base = LD->getBasePtr();
1675 SDValue Ops[] = {Base, Offset,
1689 SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1689 SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1691 SDValue RegClass =
1693 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1694 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1695 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1700 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1700 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1702 SDValue RegClass =
1704 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1705 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1706 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1711 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1711 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1713 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1715 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1716 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1717 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1722 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1722 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1724 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1726 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1727 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1728 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1733 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1733 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1734 SDValue V2, SDValue V3) {
1734 SDValue V2, SDValue V3) {
1736 SDValue RegClass =
1738 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1739 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1740 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1741 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1742 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1748 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1748 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1749 SDValue V2, SDValue V3) {
1749 SDValue V2, SDValue V3) {
1751 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1753 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1754 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1755 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1756 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1757 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1763 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1763 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1764 SDValue V2, SDValue V3) {
1764 SDValue V2, SDValue V3) {
1766 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1768 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1769 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1770 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1771 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1772 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1780 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1780 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1916 static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
1928 SDValue MemAddr, Align;
1935 SDValue Chain = N->getOperand(0);
1975 SDValue Pred = getAL(CurDAG, dl);
1976 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1978 SmallVector<SDValue, 7> Ops;
1987 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2012 SDValue ImplDef =
2014 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
2023 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2046 SDValue SuperReg = SDValue(VLd, 0);
2067 SDValue MemAddr, Align;
2077 SDValue Chain = N->getOperand(0);
2107 SDValue Pred = getAL(CurDAG, dl);
2108 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2109 SmallVector<SDValue, 7> Ops;
2113 SDValue SrcReg;
2118 SDValue V0 = N->getOperand(Vec0Idx + 0);
2119 SDValue V1 = N->getOperand(Vec0Idx + 1);
2123 SDValue V2 = N->getOperand(Vec0Idx + 2);
2126 SDValue V3 = (NumVecs == 3)
2133 SDValue Q0 = N->getOperand(Vec0Idx);
2134 SDValue Q1 = N->getOperand(Vec0Idx + 1);
2143 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2174 SDValue V0 = N->getOperand(Vec0Idx + 0);
2175 SDValue V1 = N->getOperand(Vec0Idx + 1);
2176 SDValue V2 = N->getOperand(Vec0Idx + 2);
2177 SDValue V3 = (NumVecs == 3)
2180 SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2184 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2195 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2218 SDValue MemAddr, Align;
2228 SDValue Chain = N->getOperand(0);
2277 SDValue Pred = getAL(CurDAG, dl);
2278 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2280 SmallVector<SDValue, 8> Ops;
2284 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2290 SDValue SuperReg;
2291 SDValue V0 = N->getOperand(Vec0Idx + 0);
2292 SDValue V1 = N->getOperand(Vec0Idx + 1);
2299 SDValue V2 = N->getOperand(Vec0Idx + 2);
2300 SDValue V3 = (NumVecs == 3)
2340 SDValue PredicateMask) {
2347 SDValue PredicateMask,
2348 SDValue Inactive) {
2372 SmallVector<SDValue, 8> Ops;
2404 SmallVector<SDValue, 8> Ops;
2430 SmallVector<SDValue, 8> Ops;
2438 SDValue CarryIn = N->getOperand(FirstInputOp + 2);
2483 auto Data = SDValue(
2485 SDValue Chain = N->getOperand(0);
2487 SDValue Ops[] = {Data, N->getOperand(2), Chain};
2509 SDValue MemAddr, Align;
2514 SDValue Chain = N->getOperand(0);
2562 SDValue Pred = getAL(CurDAG, dl);
2563 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2567 SmallVector<SDValue, 6> Ops;
2575 SDValue Inc = N->getOperand(2);
2591 const SDValue OpsA[] = { MemAddr, Align, Pred, Reg0, Chain };
2596 const SDValue OpsB[] = { MemAddr, Align, Pred, Reg0, Chain };
2599 SDValue ImplDef =
2601 const SDValue OpsA[] = { MemAddr, Align, ImplDef, Pred, Reg0, Chain };
2605 SDValue SuperReg = SDValue(VLdA, 0);
2607 const SDValue OpsB[] = { MemAddr, Align, SuperReg, Pred, Reg0, Chain };
2619 SDValue SuperReg = SDValue(VLdDup, 0);
2665 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2671 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2680 SDValue ShOpc =
2683 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2690 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2713 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2715 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2735 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2737 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2756 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2758 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2780 SDValue XORSrc0 = N->getOperand(0);
2781 SDValue XORSrc1 = N->getOperand(1);
2790 SDValue ADDSrc0 = XORSrc0.getOperand(0);
2791 SDValue ADDSrc1 = XORSrc0.getOperand(1);
2792 SDValue SRASrc0 = XORSrc1.getOperand(0);
2793 SDValue SRASrc1 = XORSrc1.getOperand(1);
2822 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2855 SDValue And = N->getOperand(0);
2859 SDValue Zero = N->getOperand(1);
2863 SDValue X = And.getOperand(0);
2879 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2884 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
2939 SDValue Ptr = ST->getBasePtr();
2949 SDValue Ops[] = {ST->getValue(),
2988 SDValue CPIdx = CurDAG->getTargetConstantPool(
2994 SDValue Ops[] = {
3003 SDValue Ops[] = {
3033 SDValue TFI = CurDAG->getTargetFrameIndex(
3047 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3073 SDValue V = N->getOperand(0);
3075 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3076 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3078 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
3082 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
3092 SDValue V = N->getOperand(0);
3094 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3095 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3097 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
3101 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
3132 SDValue NewImm =
3140 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3146 SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
3169 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3174 SDValue N2 = N0.getOperand(1);
3183 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3185 SDValue Ops[] = { N0.getOperand(0), Imm16,
3196 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
3205 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3212 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3224 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3231 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3255 SDValue SmulLoHi = N->getOperand(1);
3256 SDValue Subc = N->getOperand(2);
3266 SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
3288 SDValue Ops[] = { N->getOperand(1),
3299 SDValue Ops[] = { N->getOperand(1),
3324 SDValue Chain = N->getOperand(0);
3325 SDValue N1 = N->getOperand(1);
3326 SDValue N2 = N->getOperand(2);
3327 SDValue N3 = N->getOperand(3);
3328 SDValue InFlag = N->getOperand(4);
3337 SDValue Int = InFlag.getOperand(0);
3342 SDValue Elements = Int.getOperand(2);
3343 SDValue Size = CurDAG->getTargetConstant(
3347 SDValue Args[] = { Elements, Size, Int.getOperand(0) };
3354 SDValue EndArgs[] = { SDValue(LoopDec, 0), N1, Chain };
3383 SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
3384 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
3403 SDValue X = N->getOperand(0);
3415 SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3421 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
3428 SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
3437 SDValue InFlag = N->getOperand(4);
3444 SDValue ARMcc = N->getOperand(2);
3457 SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
3458 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
3485 SDValue Pred = getAL(CurDAG, dl);
3486 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3487 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3508 SDValue Pred = getAL(CurDAG, dl);
3509 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3510 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3530 SDValue Pred = getAL(CurDAG, dl);
3531 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3532 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3806 SDValue Chain = N->getOperand(0);
3814 SmallVector<SDValue, 5> Ops;
3838 SDValue Chain = N->getOperand(0);
3839 SDValue MemAddr = N->getOperand(2);
3856 SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
3864 SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3866 SDValue Result;
3870 SDValue SubRegIdx =
3879 SDValue Result;
3883 SDValue SubRegIdx =
3898 SDValue Chain = N->getOperand(0);
3899 SDValue Val0 = N->getOperand(2);
3900 SDValue Val1 = N->getOperand(3);
3901 SDValue MemAddr = N->getOperand(4);
3909 SmallVector<SDValue, 7> Ops;
4298 std::vector<SDValue> &Ops) {
4420 std::vector<SDValue> Ops;
4496 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4535 std::vector<SDValue> Ops;
4551 SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
4607 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4631 std::vector<SDValue> AsmNodeOperands;
4645 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
4651 SDValue op = N->getOperand(i);
4669 SDValue op = N->getOperand(++i);
4691 SDValue op = N->getOperand(++i);
4707 SDValue V0 = N->getOperand(i+1);
4708 SDValue V1 = N->getOperand(i+2);
4711 SDValue PairedReg;
4721 SDValue Chain = SDValue(N,0);
4724 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
4728 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4730 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4732 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
4734 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
4737 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
4744 SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
4747 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
4749 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
4751 SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
4787 SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
4796 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
4797 std::vector<SDValue> &OutOps) {
lib/Target/ARM/ARMISelLowering.cpp 1762 static bool isSRL16(const SDValue &Op) {
1770 static bool isSRA16(const SDValue &Op) {
1778 static bool isSHL16(const SDValue &Op) {
1790 static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
1925 SDValue ARMTargetLowering::LowerCallResult(
1926 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
1926 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
1928 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1929 SDValue ThisVal) const {
1949 SDValue Val;
1952 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
1957 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
1966 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
2006 SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
2006 SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
2006 SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
2007 SDValue Arg, const SDLoc &dl,
2012 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
2021 SDValue Chain, SDValue &Arg,
2021 SDValue Chain, SDValue &Arg,
2024 SDValue &StackPtr,
2025 SmallVectorImpl<SDValue> &MemOpChains,
2027 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2049 SDValue
2051 SmallVectorImpl<SDValue> &InVals) const {
2055 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2057 SDValue Chain = CLI.Chain;
2058 SDValue Callee = CLI.Callee;
2122 SDValue StackPtr =
2126 SmallVector<SDValue, 8> MemOpChains;
2134 SDValue Arg = OutVals[realArgIdx];
2159 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2161 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2212 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
2213 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
2214 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
2231 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
2232 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
2233 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
2234 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
2235 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
2237 SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), dl,
2241 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2258 SDValue InFlag;
2296 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
2310 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
2360 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
2365 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2391 std::vector<SDValue> Ops;
2428 SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
2508 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2534 SDValue Ptr = Ld->getBasePtr();
2552 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
2555 const SmallVectorImpl<SDValue> &OutVals,
2641 SDValue Arg = OutVals[realArgIdx];
2686 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2686 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2719 SDValue
2720 ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2723 const SmallVectorImpl<SDValue> &OutVals,
2735 SDValue Flag;
2736 SmallVector<SDValue, 4> RetOps;
2751 SDValue Arg = OutVals[realRVLocIdx];
2766 SDValue ZE = Arg.getOperand(0);
2768 SDValue BC = ZE.getOperand(0);
2789 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2791 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
2813 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2868 bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2874 SDValue TCChain = Chain;
2897 SDValue UseChain = UI->getOperand(0);
2957 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
2957 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
2959 SDValue WriteValue = Op->getOperand(2);
2965 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
2967 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
2969 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
2979 SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
2979 SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
2985 SDValue Res;
3003 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV),
3021 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3021 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3029 SDValue CPAddr;
3042 SDValue Result = DAG.getLoad(
3047 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
3075 SDValue
3076 ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3084 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3088 SDValue Chain = DAG.getEntryNode();
3089 SDValue FuncTLVGet = DAG.getLoad(
3120 SDValue
3121 ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3125 SDValue Chain = DAG.getEntryNode();
3130 SDValue Ops[] = {Chain,
3137 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
3140 SDValue TEB = CurrentTEB.getValue(0);
3145 SDValue TLSArray =
3153 SDValue TLSIndex =
3158 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
3160 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
3167 SDValue Offset = DAG.getLoad(
3176 SDValue
3188 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4);
3193 SDValue Chain = Argument.getValue(1);
3195 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3211 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3211 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3217 SDValue
3223 SDValue Offset;
3224 SDValue Chain = DAG.getEntryNode();
3227 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3246 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3269 SDValue
3270 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3317 static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3403 SDValue CPAddr =
3423 SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3423 SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3436 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3436 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3446 if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
3451 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3453 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3461 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3462 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3466 SDValue RelAddr;
3469 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
3474 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
3480 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
3481 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
3494 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4);
3502 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3502 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3518 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
3519 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
3527 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3527 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3543 SDValue Result;
3559 SDValue
3560 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3562 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
3568 SDValue
3569 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3575 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3575 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3582 SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3583 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3595 SDValue Chain = Op.getOperand(0);
3603 SDValue ReturnAddress =
3606 SDValue Callee =
3608 SDValue RegisterMask = DAG.getRegisterMask(Mask);
3624 SDValue
3625 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3636 const SDValue &Operand = Op.getOperand(1);
3638 SDValue SRA =
3640 SDValue XOR = DAG.getNode(ISD::XOR, dl, VTy, SRA, Operand);
3641 SDValue SHL =
3643 SDValue OR =
3645 SDValue Result = DAG.getNode(ISD::CTLZ, dl, VTy, OR);
3651 const SDValue &Operand = Op.getOperand(1);
3654 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
3656 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
3658 SDValue Constant0 = DAG.getConstant(0, dl, VTy);
3659 SDValue Constant1 = DAG.getConstant(1, dl, VTy);
3660 SDValue Constant31 = DAG.getConstant(31, dl, VTy);
3661 SDValue SRAHi = DAG.getNode(ISD::SRA, dl, VTy, Hi, Constant31);
3662 SDValue XORHi = DAG.getNode(ISD::XOR, dl, VTy, SRAHi, Hi);
3663 SDValue SHLHi = DAG.getNode(ISD::SHL, dl, VTy, XORHi, Constant1);
3664 SDValue ORHi = DAG.getNode(ISD::OR, dl, VTy, SHLHi, Constant1);
3665 SDValue CLSHi = DAG.getNode(ISD::CTLZ, dl, VTy, ORHi);
3666 SDValue CheckLo =
3668 SDValue HiIsZero =
3670 SDValue AdjustedLo =
3672 SDValue CLZAdjustedLo = DAG.getNode(ISD::CTLZ, dl, VTy, AdjustedLo);
3673 SDValue Result =
3683 SDValue CPAddr;
3691 SDValue Result = DAG.getLoad(
3696 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3754 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
3754 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
3791 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
3791 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
3818 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
3818 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
3826 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
3832 SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
3834 SDValue &Root,
3848 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
3850 SDValue ArgValue2;
3856 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3878 const SDLoc &dl, SDValue &Chain,
3910 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
3912 SmallVector<SDValue, 4> MemOps;
3918 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
3919 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
3932 const SDLoc &dl, SDValue &Chain,
3951 SDValue ARMTargetLowering::LowerFormalArguments(
3952 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3954 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3966 SmallVector<SDValue, 16> ArgValues;
3967 SDValue ArgValue;
4027 SDValue ArgValue1 = GetF64FormalArgument(VA, ArgLocs[++i],
4030 SDValue ArgValue2;
4033 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4134 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4156 static bool isFloatingPointZero(SDValue Op) {
4162 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
4171 SDValue BitcastOp = Op->getOperand(0);
4181 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4181 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4181 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4182 SDValue &ARMcc, SelectionDAG &DAG,
4259 SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
4279 SDValue Shift = DAG.getNode(ARMISD::LSLS, dl,
4283 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
4322 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4322 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4322 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4325 SDValue Cmp;
4335 SDValue
4336 ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
4358 std::pair<SDValue, SDValue>
4358 std::pair<SDValue, SDValue>
4359 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4360 SDValue &ARMcc) const {
4363 SDValue Value, OverflowCmp;
4364 SDValue LHS = Op.getOperand(0);
4365 SDValue RHS = Op.getOperand(1);
4428 SDValue
4429 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
4434 SDValue Value, OverflowCmp;
4435 SDValue ARMcc;
4437 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4440 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4441 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4444 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
4451 static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,
4451 static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,
4458 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL,
4464 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
4464 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
4475 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
4475 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
4481 SDValue LHS = Op.getOperand(0);
4482 SDValue RHS = Op.getOperand(1);
4487 SDValue Value;
4488 SDValue Overflow;
4512 static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4512 static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4534 SDValue Add =
4541 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
4541 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
4542 SDValue Cond = Op.getOperand(0);
4543 SDValue SelectTrue = Op.getOperand(1);
4544 SDValue SelectFalse = Op.getOperand(2);
4554 SDValue Value, OverflowCmp;
4555 SDValue ARMcc;
4557 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4579 SDValue True;
4580 SDValue False;
4591 SDValue ARMcc = Cond.getOperand(2);
4592 SDValue CCR = Cond.getOperand(3);
4593 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
4660 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
4660 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
4661 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
4661 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
4661 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
4662 SDValue Cmp, SelectionDAG &DAG) const {
4669 SDValue TrueLow = TrueVal.getValue(0);
4670 SDValue TrueHigh = TrueVal.getValue(1);
4671 SDValue FalseLow = FalseVal.getValue(0);
4672 SDValue FalseHigh = FalseVal.getValue(1);
4674 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
4676 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
4700 static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
4700 static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
4701 const SDValue TrueVal, const SDValue FalseVal,
4701 const SDValue TrueVal, const SDValue FalseVal,
4702 const ISD::CondCode CC, const SDValue K) {
4710 static bool isUpperSaturate(const SDValue LHS, const SDValue RHS,
4710 static bool isUpperSaturate(const SDValue LHS, const SDValue RHS,
4711 const SDValue TrueVal, const SDValue FalseVal,
4711 const SDValue TrueVal, const SDValue FalseVal,
4712 const ISD::CondCode CC, const SDValue K) {
4736 static bool isSaturatingConditional(const SDValue &Op, SDValue &V,
4736 static bool isSaturatingConditional(const SDValue &Op, SDValue &V,
4738 SDValue LHS1 = Op.getOperand(0);
4739 SDValue RHS1 = Op.getOperand(1);
4740 SDValue TrueVal1 = Op.getOperand(2);
4741 SDValue FalseVal1 = Op.getOperand(3);
4744 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
4748 SDValue LHS2 = Op2.getOperand(0);
4749 SDValue RHS2 = Op2.getOperand(1);
4750 SDValue TrueVal2 = Op2.getOperand(2);
4751 SDValue FalseVal2 = Op2.getOperand(3);
4756 SDValue *K1 = isa<ConstantSDNode>(LHS1) ? &LHS1 : isa<ConstantSDNode>(RHS1)
4759 SDValue *K2 = isa<ConstantSDNode>(LHS2) ? &LHS2 : isa<ConstantSDNode>(RHS2)
4762 SDValue K2Tmp = isa<ConstantSDNode>(TrueVal2) ? TrueVal2 : FalseVal2;
4763 SDValue V1Tmp = (K1 && *K1 == LHS1) ? RHS1 : LHS1;
4764 SDValue V2Tmp = (K2 && *K2 == LHS2) ? RHS2 : LHS2;
4765 SDValue V2 = (K2Tmp == TrueVal2) ? FalseVal2 : TrueVal2;
4771 SDValue V2TmpReg = V2Tmp;
4782 const SDValue *LowerCheckOp =
4788 const SDValue *UpperCheckOp =
4835 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
4835 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
4836 SDValue &SatK)
4838 SDValue LHS = Op.getOperand(0);
4839 SDValue RHS = Op.getOperand(1);
4841 SDValue TrueVal = Op.getOperand(2);
4842 SDValue FalseVal = Op.getOperand(3);
4844 SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS)
4852 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
4854 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
4879 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
4879 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
4884 SDValue SatValue;
4903 SDValue LowerSatConstant;
4906 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
4909 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
4916 SDValue LHS = Op.getOperand(0);
4917 SDValue RHS = Op.getOperand(1);
4919 SDValue TrueVal = Op.getOperand(2);
4920 SDValue FalseVal = Op.getOperand(3);
4967 SDValue ARMcc;
4968 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5008 SDValue ARMcc;
5009 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5010 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5042 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5043 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5044 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5045 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5047 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
5049 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5057 static bool canChangeToInt(SDValue Op, bool &SeenZero,
5078 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5078 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5090 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5091 SDValue &RetVal1, SDValue &RetVal2) {
5091 SDValue &RetVal1, SDValue &RetVal2) {
5101 SDValue Ptr = Ld->getBasePtr();
5108 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
5121 SDValue
5122 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5123 SDValue Chain = Op.getOperand(0);
5125 SDValue LHS = Op.getOperand(2);
5126 SDValue RHS = Op.getOperand(3);
5127 SDValue Dest = Op.getOperand(4);
5143 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5144 SDValue ARMcc;
5150 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5151 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5156 SDValue LHS1, LHS2;
5157 SDValue RHS1, RHS2;
5165 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5172 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5172 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5173 SDValue Chain = Op.getOperand(0);
5174 SDValue Cond = Op.getOperand(1);
5175 SDValue Dest = Op.getOperand(2);
5191 SDValue Value, OverflowCmp;
5192 SDValue ARMcc;
5200 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5209 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5209 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5210 SDValue Chain = Op.getOperand(0);
5212 SDValue LHS = Op.getOperand(2);
5213 SDValue RHS = Op.getOperand(3);
5214 SDValue Dest = Op.getOperand(4);
5243 SDValue Value, OverflowCmp;
5244 SDValue ARMcc;
5254 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5261 SDValue ARMcc;
5262 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5263 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5271 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5278 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5279 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5280 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5282 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
5283 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
5286 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
5292 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5292 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5293 SDValue Chain = Op.getOperand(0);
5294 SDValue Table = Op.getOperand(1);
5295 SDValue Index = Op.getOperand(2);
5300 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
5303 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
5328 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5328 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5359 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5359 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5379 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5379 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5424 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5424 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5444 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5444 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5446 SDValue Tmp0 = Op.getOperand(0);
5447 SDValue Tmp1 = Op.getOperand(1);
5458 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
5480 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff),
5483 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
5486 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
5507 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
5508 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5520 SDValue Lo = Tmp0.getValue(0);
5521 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
5526 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
5526 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
5538 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5539 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
5550 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
5550 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
5561 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
5583 static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
5591 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
5608 static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
5610 SDValue Op = BC->getOperand(0);
5642 SDValue ExtractSrc = Op.getOperand(0);
5646 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
5656 static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
5660 SDValue Op = N->getOperand(0);
5683 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) };
5684 SDValue Copy = DAG.getNode(ISD::CopyFromReg, SDLoc(Op), MVT::f16, Ops);
5729 SDValue Cvt = DAG.getNode(ARMISD::VMOVrh, SDLoc(Op), MVT::i32, Op);
5743 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
5746 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
5748 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
5756 SDValue Cvt;
5778 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
5781 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
5783 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
5789 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
5789 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
5795 SDValue ShOpLo = Op.getOperand(0);
5796 SDValue ShOpHi = Op.getOperand(1);
5797 SDValue ShAmt = Op.getOperand(2);
5798 SDValue ARMcc;
5799 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5804 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
5806 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
5807 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
5809 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
5810 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
5811 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
5812 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
5814 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift,
5817 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
5818 SDValue HiBigShift = Opc == ISD::SRA
5822 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
5824 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
5827 SDValue Ops[2] = { Lo, Hi };
5833 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
5833 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
5839 SDValue ShOpLo = Op.getOperand(0);
5840 SDValue ShOpHi = Op.getOperand(1);
5841 SDValue ShAmt = Op.getOperand(2);
5842 SDValue ARMcc;
5843 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5846 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
5848 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
5849 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
5850 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
5852 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
5854 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
5855 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
5857 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
5860 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
5862 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
5863 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
5866 SDValue Ops[2] = { Lo, Hi };
5870 SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
5870 SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
5877 SDValue Ops[] = { DAG.getEntryNode(),
5880 SDValue FPSCR = DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, MVT::i32, Ops);
5881 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
5883 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
5889 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
5896 SDValue X = N->getOperand(0);
5897 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
5898 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
5904 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
5906 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
5914 SDValue WidthMinus1 =
5917 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
5924 SDValue Bits;
5927 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
5931 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
5941 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
5945 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
5957 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
5964 SmallVector<SDValue, 8> Ops;
5981 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6002 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6016 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6031 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6067 SDValue NegatedCount = DAG.getNode(
6074 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6089 SDValue ShAmt = N->getOperand(1);
6116 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6119 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6139 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6141 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6156 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6156 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6162 SDValue Op0 = Op.getOperand(0);
6163 SDValue Op1 = Op.getOperand(1);
6164 SDValue CC = Op.getOperand(2);
6193 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
6194 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
6195 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
6197 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6198 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
6237 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6239 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6241 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6249 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6251 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6253 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6282 SDValue AndOp;
6295 SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
6308 SDValue SingleOp;
6319 SDValue Result;
6336 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6336 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6337 SDValue LHS = Op.getOperand(0);
6338 SDValue RHS = Op.getOperand(1);
6339 SDValue Carry = Op.getOperand(2);
6340 SDValue Cond = Op.getOperand(3);
6353 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
6355 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
6356 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
6357 SDValue ARMcc = DAG.getConstant(
6359 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6360 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
6369 static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6509 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
6509 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
6530 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
6531 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
6563 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
6564 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
6585 SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
6589 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
6595 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
6606 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
6612 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7014 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7014 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7031 static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7031 static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7055 SDValue FirstOp = Op.getOperand(0);
7061 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
7069 SDValue V = Op.getOperand(i);
7078 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
7081 SDValue V = Op.getOperand(i);
7093 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7093 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7113 SDValue Val = isVMOVModifiedImm(SplatBits.getZExtValue(),
7119 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
7130 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
7138 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
7160 DenseMap<SDValue, unsigned> ValueCounts;
7161 SDValue Value;
7163 SDValue V = Op.getOperand(i);
7199 SDValue N;
7232 SmallVector<SDValue, 3> Ops;
7242 SmallVector<SDValue, 8> Ops;
7250 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
7256 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
7270 SDValue shuffle = ReconstructShuffle(Op, DAG);
7278 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7281 SDValue Lower =
7285 SDValue Upper = DAG.getBuildVector(
7301 SmallVector<SDValue, 8> Ops;
7304 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
7315 SDValue Vec = DAG.getUNDEF(VT);
7317 SDValue V = Op.getOperand(i);
7320 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
7331 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7331 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7339 SDValue Vec;
7346 SDValue ShuffleVec;
7353 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7355 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7362 SDValue V = Op.getOperand(i);
7376 SDValue SourceVec = V.getOperand(0);
7452 SDValue VEXTSrc1 =
7455 SDValue VEXTSrc2 =
7488 SDValue Entry = Op.getOperand(i);
7518 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
7522 SDValue Shuffle = buildLegalVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
7614 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
7614 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
7615 SDValue RHS, SelectionDAG &DAG,
7627 SDValue OpLHS, OpRHS;
7672 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
7672 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
7676 SDValue V1 = Op.getOperand(0);
7677 SDValue V2 = Op.getOperand(1);
7680 SmallVector<SDValue, 8> VTBLMask;
7693 static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
7693 static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
7696 SDValue OpLHS = Op.getOperand(0);
7723 static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
7723 static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
7728 SDValue AllOnes =
7732 SDValue AllZeroes =
7739 SDValue RecastV1;
7750 SDValue PredAsVector =
7758 static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
7758 static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
7767 SDValue V1 = Op.getOperand(0);
7770 SDValue cast = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, V1);
7771 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast);
7772 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit,
7785 SDValue PredAsVector = PromoteMVEPredVector(dl, V1, VT, DAG);
7789 SDValue Shuffled = DAG.getVectorShuffle(NewVT, dl, PredAsVector,
7798 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
7798 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
7800 SDValue V1 = Op.getOperand(0);
7801 SDValue V2 = Op.getOperand(1);
7908 SDValue SubV1 = V1->getOperand(0);
7909 SDValue SubV2 = V1->getOperand(1);
7923 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
7971 SmallVector<SDValue, 8> Ops;
7981 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
7989 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
7995 static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
7995 static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8003 SDValue Conv =
8009 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32,
8011 SDValue BFI = DAG.getNode(ARMISD::BFI, dl, MVT::i32, Conv, Ext,
8016 SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8016 SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8019 SDValue Lane = Op.getOperand(2);
8023 SDValue Elt = Op.getOperand(1);
8043 SDValue VecIn = Op.getOperand(0);
8048 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt);
8049 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn);
8050 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT,
8058 static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8058 static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8066 SDValue Conv =
8071 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv,
8076 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8076 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8079 SDValue Lane = Op.getOperand(1);
8083 SDValue Vec = Op.getOperand(0);
8097 static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8097 static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8099 SDValue V1 = Op.getOperand(0);
8100 SDValue V2 = Op.getOperand(1);
8113 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8114 SDValue NewV2 = PromoteMVEPredVector(dl, V2, Op2VT, DAG);
8127 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
8132 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
8149 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
8149 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
8160 SDValue Val = DAG.getUNDEF(MVT::v2f64);
8161 SDValue Op0 = Op.getOperand(0);
8162 SDValue Op1 = Op.getOperand(1);
8174 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
8174 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
8176 SDValue V1 = Op.getOperand(0);
8177 SDValue V2 = Op.getOperand(1);
8189 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8197 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
8199 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
8305 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
8305 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
8327 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
8351 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
8362 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
8365 SDValue extLoad =
8389 SmallVector<SDValue, 8> Ops;
8423 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
8423 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
8470 SDValue Op0;
8471 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
8488 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
8489 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
8498 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
8498 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
8498 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
8529 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
8529 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
8529 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
8533 SDValue N2;
8568 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
8568 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
8575 SDValue N0 = Op.getOperand(0);
8576 SDValue N1 = Op.getOperand(1);
8577 SDValue N2, N3;
8604 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
8604 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
8612 SDValue N0 = Op.getOperand(0);
8613 SDValue N1 = Op.getOperand(1);
8614 SDValue N2, N3;
8648 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
8681 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
8681 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
8686 SDValue Carry = Op.getOperand(2);
8690 SDValue Result;
8725 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
8725 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
8731 SDValue Arg = Op.getOperand(0);
8745 SDValue SRet;
8774 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
8781 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
8781 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
8786 SDValue LoadSin =
8790 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
8792 SDValue LoadCos =
8800 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
8800 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
8802 SDValue &Chain) const {
8817 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
8841 SDValue
8880 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
8880 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
8886 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
8892 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
8892 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
8894 SDValue Op = N->getOperand(1);
8897 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
8899 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
8906 SDValue Op, SelectionDAG &DAG, bool Signed,
8907 SmallVectorImpl<SDValue> &Results) const {
8915 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode());
8917 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
8919 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
8920 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
8928 static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
8928 static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
8947 SDValue Load = DAG.getExtLoad(
8951 SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Load);
8958 static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
8958 static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
8970 SDValue Build = ST->getValue();
8972 SmallVector<SDValue, 16> Ops;
8980 SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
8987 static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
8987 static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
8990 SDValue Mask = N->getMask();
8991 SDValue PassThru = N->getPassThru();
9005 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
9007 SDValue NewLoad = DAG.getMaskedLoad(
9010 SDValue Combo = NewLoad;
9018 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
9018 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
9029 SmallVectorImpl<SDValue> &Results,
9035 SDValue Ops[] = { N->getOperand(0), // Chain
9044 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
9051 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
9051 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
9053 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32);
9054 SDValue VHi = DAG.getAnyExtOrTrunc(
9060 SDValue RegClass =
9062 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
9063 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
9064 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
9070 SmallVectorImpl<SDValue> & Results,
9074 SDValue Ops[] = {N->getOperand(1),
9096 static SDValue LowerFPOWI(SDValue Op, const ARMSubtarget &Subtarget,
9096 static SDValue LowerFPOWI(SDValue Op, const ARMSubtarget &Subtarget,
9104 SDValue Val = Op.getOperand(0);
9106 SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, dl, Ty, Op.getOperand(1));
9107 SDValue Callee = DAG.getExternalSymbol(Ty == MVT::f32 ? "powf" : "pow",
9128 SDValue InChain = DAG.getEntryNode();
9129 SDValue TCChain = InChain;
9142 std::pair<SDValue, SDValue> CI = TLI.LowerCallTo(CLI);
9142 std::pair<SDValue, SDValue> CI = TLI.LowerCallTo(CLI);
9148 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
9148 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
9242 static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
9258 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
9261 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
9265 SDValue LongMul = DAG.getNode(Opc, dl,
9276 SmallVectorImpl<SDValue> &Results,
9278 SDValue Res;
10818 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
10835 SDValue &CC, bool &Invert,
10836 SDValue &OtherOp,
10842 SDValue N1 = N->getOperand(1);
10843 SDValue N2 = N->getOperand(2);
10908 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
10908 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
10908 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
10913 SDValue NonConstantVal;
10914 SDValue CCOp;
10921 SDValue TrueVal = OtherOp;
10922 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
10934 SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
10936 SDValue N0 = N->getOperand(0);
10937 SDValue N1 = N->getOperand(1);
10939 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
10942 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
10959 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
10959 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
10959 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
10978 SmallVector<SDValue, 8> Ops;
10987 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
10987 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
10987 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
10997 SDValue N00 = N0.getOperand(0);
10998 SDValue N10 = N1.getOperand(0);
11017 SmallVector<SDValue, 8> Ops;
11029 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT,
11039 static SDValue
11040 AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
11040 AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
11064 SDValue Vec = N0->getOperand(0)->getOperand(0);
11075 SDValue ExtVec0 = N0->getOperand(i);
11076 SDValue ExtVec1 = N1->getOperand(i);
11112 SmallVector<SDValue, 8> Ops;
11132 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
11137 static SDValue findMUL_LOHI(SDValue V) {
11137 static SDValue findMUL_LOHI(SDValue V) {
11144 static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
11154 SDValue Mul = AddcNode->getOperand(0);
11155 SDValue Lo = AddcNode->getOperand(1);
11163 SDValue SRA = AddeNode->getOperand(0);
11164 SDValue Hi = AddeNode->getOperand(1);
11183 SDValue Op0;
11184 SDValue Op1;
11207 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
11210 SDValue HiMLALResult(SMLAL.getNode(), 1);
11211 SDValue LoMLALResult(SMLAL.getNode(), 0);
11217 SDValue resNode(AddcNode, 0);
11221 static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
11260 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0);
11261 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1);
11281 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0);
11282 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1);
11290 SDValue MULOp = findMUL_LOHI(AddeSubeOp0);
11303 SDValue *HiAddSub = nullptr;
11304 SDValue *LoMul = nullptr;
11305 SDValue *LowAddSub = nullptr;
11341 SmallVector<SDValue, 8> Ops;
11360 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops);
11373 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode),
11377 SDValue HiMLALResult(MLALNode.getNode(), 1);
11380 SDValue LoMLALResult(MLALNode.getNode(), 0);
11387 static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
11406 SDValue AddHi;
11427 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1),
11429 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
11442 static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
11464 static SDValue PerformAddcSubcCombine(SDNode *N,
11471 SDValue LHS = N->getOperand(0);
11472 SDValue RHS = N->getOperand(1);
11481 SDValue RHS = N->getOperand(1);
11497 static SDValue PerformAddeSubeCombine(SDNode *N,
11502 SDValue RHS = N->getOperand(1);
11525 static SDValue PerformABSCombine(SDNode *N,
11528 SDValue res;
11544 static SDValue PerformADDECombine(SDNode *N,
11561 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
11561 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
11561 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
11565 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
11569 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
11571 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
11577 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
11597 SDValue N1 = N->getOperand(0);
11638 static SDValue PerformSHLSimplify(SDNode *N,
11698 SDValue SHL = N->getOperand(0);
11728 SDValue X = SHL.getOperand(0);
11729 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X,
11732 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1));
11743 static SDValue PerformADDCombine(SDNode *N,
11746 SDValue N0 = N->getOperand(0);
11747 SDValue N1 = N->getOperand(1);
11750 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
11754 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
11763 static SDValue PerformSUBCombine(SDNode *N,
11765 SDValue N0 = N->getOperand(0);
11766 SDValue N1 = N->getOperand(1);
11770 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI))
11791 static SDValue PerformVMULCombine(SDNode *N,
11798 SDValue N0 = N->getOperand(0);
11799 SDValue N1 = N->getOperand(1);
11815 SDValue N00 = N0->getOperand(0);
11816 SDValue N01 = N0->getOperand(1);
11822 static SDValue PerformMULCombine(SDNode *N,
11847 SDValue V = N->getOperand(0);
11850 SDValue Res;
11905 static SDValue CombineANDShift(SDNode *N,
11959 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
11970 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
11983 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
11996 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
12009 static SDValue PerformANDCombine(SDNode *N,
12028 SDValue Val = isVMOVModifiedImm((~SplatBits).getZExtValue(),
12033 SDValue Input =
12035 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
12043 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI))
12046 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
12051 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
12058 static SDValue PerformORCombineToSMULWBT(SDNode *OR,
12066 SDValue SRL = OR->getOperand(0);
12067 SDValue SHL = OR->getOperand(1);
12093 SDValue OpS16 = SMULLOHI->getOperand(0);
12094 SDValue OpS32 = SMULLOHI->getOperand(1);
12113 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16);
12118 static SDValue PerformORCombineToBFI(SDNode *N,
12126 SDValue N0 = N->getOperand(0);
12127 SDValue N1 = N->getOperand(1);
12143 SDValue N00 = N0.getOperand(0);
12148 SDValue MaskOp = N0.getOperand(1);
12155 SDValue Res;
12226 SDValue ShAmt = N00.getOperand(1);
12261 static SDValue PerformORCombine_i1(SDNode *N,
12267 SDValue N0 = N->getOperand(0);
12268 SDValue N1 = N->getOperand(1);
12297 SmallVector<SDValue, 4> Ops0;
12302 SmallVector<SDValue, 4> Ops1;
12308 SDValue NewN0 = DCI.DAG.getNode(N0->getOpcode(), SDLoc(N0), VT, Ops0);
12309 SDValue NewN1 = DCI.DAG.getNode(N1->getOpcode(), SDLoc(N1), VT, Ops1);
12310 SDValue And = DCI.DAG.getNode(ISD::AND, SDLoc(N), VT, NewN0, NewN1);
12316 static SDValue PerformORCombine(SDNode *N,
12335 SDValue Val = isVMOVModifiedImm(SplatBits.getZExtValue(),
12340 SDValue Input =
12342 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
12350 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
12352 if (SDValue Result = PerformORCombineToSMULWBT(N, DCI, Subtarget))
12356 SDValue N0 = N->getOperand(0);
12357 SDValue N1 = N->getOperand(1);
12389 SDValue Result = DAG.getNode(ARMISD::VBSL, dl, CanonicalVT,
12406 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
12410 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
12416 static SDValue PerformXORCombine(SDNode *N,
12427 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
12430 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
12440 static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
12443 SDValue From = N->getOperand(1);
12469 static SDValue FindBFIToCombineWith(SDNode *N) {
12473 SDValue From = ParseBFI(N, ToMask, FromMask);
12474 SDValue To = N->getOperand(0);
12479 SDValue V = To;
12483 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
12513 static SDValue PerformBFICombine(SDNode *N,
12515 SDValue N1 = N->getOperand(1);
12538 SDValue CombineBFI = FindBFIToCombineWith(N);
12544 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
12547 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
12572 static SDValue PerformVMOVRRDCombine(SDNode *N,
12576 SDValue InDouble = N->getOperand(0);
12591 SDValue BasePtr = LD->getBasePtr();
12592 SDValue NewLD1 =
12596 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
12599 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr,
12607 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
12616 static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
12618 SDValue Op0 = N->getOperand(0);
12619 SDValue Op1 = N->getOperand(1);
12648 static SDValue PerformBUILD_VECTORCombine(SDNode *N,
12657 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
12666 SmallVector<SDValue, 8> Ops;
12669 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
12675 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
12680 static SDValue
12719 SDValue Elt = N->getOperand(Idx);
12747 SDValue Vec = DAG.getUNDEF(VecVT);
12750 SDValue V = N->getOperand(Idx);
12762 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
12771 static SDValue
12774 SDValue Op = N->getOperand(0);
12791 static SDValue PerformInsertEltCombine(SDNode *N,
12805 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
12806 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
12810 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
12817 static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
12828 SDValue Op0 = N->getOperand(0);
12829 SDValue Op1 = N->getOperand(1);
12835 SDValue Concat0Op1 = Op0.getOperand(1);
12836 SDValue Concat1Op1 = Op1.getOperand(1);
12847 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
12872 static SDValue CombineBaseUpdate(SDNode *N,
12879 SDValue Addr = N->getOperand(AddrOpIdx);
12978 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
13038 SmallVector<SDValue, 8> Ops;
13059 SDValue &StVal = Ops[Ops.size()-2];
13064 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
13068 SmallVector<SDValue, 5> NewResults;
13075 SDValue &LdVal = NewResults[0];
13088 static SDValue PerformVLDCombine(SDNode *N,
13149 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
13151 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
13168 std::vector<SDValue> VLDDupResults;
13179 static SDValue PerformVDUPLANECombine(SDNode *N,
13181 SDValue Op = N->getOperand(0);
13210 static SDValue PerformVDUPCombine(SDNode *N,
13214 SDValue Op = N->getOperand(0);
13225 SDValue Ops[] = { LD->getOperand(0), LD->getOperand(1),
13228 SDValue VLDDup = DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys,
13238 static SDValue PerformLOADCombine(SDNode *N,
13253 static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
13255 SDValue StVal = St->getValue();
13284 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
13294 SDValue Shuff = DAG.getVectorShuffle(
13314 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
13315 SmallVector<SDValue, 8> Chains;
13316 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
13318 SDValue BasePtr = St->getBasePtr();
13323 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
13325 SDValue Ch =
13338 static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
13342 SDValue Trunc = St->getValue();
13364 SDValue Ch = St->getChain();
13365 SDValue BasePtr = St->getBasePtr();
13373 SmallVector<SDValue, 4> Stores;
13376 SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
13378 SDValue Extract =
13381 SDValue Store = DAG.getTruncStore(
13391 static SDValue PerformSTORECombine(SDNode *N,
13397 SDValue StVal = St->getValue();
13401 if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG))
13405 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
13418 SDValue BasePtr = St->getBasePtr();
13419 SDValue NewST1 = DAG.getStore(
13424 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
13440 SDValue IntVec = StVal.getOperand(0);
13443 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
13444 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
13447 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
13474 static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
13479 SDValue Op = N->getOperand(0);
13484 SDValue ConstVec = Op->getOperand(1);
13511 SDValue FixConv = DAG.getNode(
13531 static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG,
13536 SDValue Op = N->getOperand(0);
13542 SDValue ConstVec = N->getOperand(1);
13567 SDValue ConvInput = Op.getOperand(0);
13582 static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
13741 static SDValue PerformShiftCombine(SDNode *N,
13749 SDValue N1 = N->getOperand(1);
13751 SDValue N0 = N->getOperand(0);
13768 SDValue N0 = N->getOperand(0);
13784 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
13829 static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
13830 SDValue N0 = N->getOperand(0);
13858 SDValue Ch = LD->getChain();
13859 SDValue BasePtr = LD->getBasePtr();
13866 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
13870 SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
13876 SDValue NewLoad1 =
13879 SDValue NewLoad2 =
13884 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
13893 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
13895 SDValue N0 = N->getOperand(0);
13902 SDValue Vec = N0.getOperand(0);
13903 SDValue Lane = N0.getOperand(1);
13929 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
13935 static const APInt *isPowerOf2Constant(SDValue V) {
13943 SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
13957 SDValue Op0 = CMOV->getOperand(0);
13958 SDValue Op1 = CMOV->getOperand(1);
13961 SDValue CmpZ = CMOV->getOperand(4);
13968 SDValue And = CmpZ->getOperand(0);
13974 SDValue X = And->getOperand(0);
13990 SDValue Y = Op1->getOperand(0);
14008 SDValue V = Y;
14039 static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
14039 static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
14076 static SDValue PerformHWLoopCombine(SDNode *N,
14092 SDValue Cond;
14095 SDValue Chain = N->getOperand(0);
14096 SDValue Dest;
14115 SDValue Int = SearchLoopIntrinsic(Cond, CC, Imm, Negate);
14143 SDValue Elements = Int.getOperand(2);
14148 SDValue OtherTarget = Br->getOperand(1);
14152 SDValue NewBrOps[] = { Br->getOperand(0), Dest };
14153 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps);
14158 SDValue Res;
14161 SDValue Ops[] = { Chain, Elements, Dest };
14168 SDValue Ops[] = { Chain, Elements, OtherTarget };
14174 SDValue Size = DAG.getTargetConstant(
14176 SDValue Args[] = { Int.getOperand(0), Elements, Size, };
14177 SDValue LoopDec = DAG.getNode(ARMISD::LOOP_DEC, dl,
14182 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
14192 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
14199 SDValue
14201 SDValue Cmp = N->getOperand(4);
14208 SDValue LHS = Cmp.getOperand(0);
14209 SDValue RHS = Cmp.getOperand(1);
14210 SDValue Chain = N->getOperand(0);
14211 SDValue BB = N->getOperand(1);
14212 SDValue ARMcc = N->getOperand(2);
14239 SDValue
14241 SDValue Cmp = N->getOperand(4);
14248 SDValue LHS = Cmp.getOperand(0);
14249 SDValue RHS = Cmp.getOperand(1);
14250 SDValue FalseVal = N->getOperand(0);
14251 SDValue TrueVal = N->getOperand(1);
14252 SDValue ARMcc = N->getOperand(2);
14258 SDValue R = PerformCMOVToBFICombine(N, DAG);
14280 SDValue Res;
14285 SDValue ARMcc;
14286 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
14316 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
14329 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
14331 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
14334 SDValue Carry =
14343 SDValue Sub =
14345 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
14357 SDValue Sub =
14359 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
14391 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
14416 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
14692 bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
14809 bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
15124 bool isSEXTLoad, SDValue &Base,
15125 SDValue &Offset, bool &isInc,
15183 bool isSEXTLoad, SDValue &Base,
15184 SDValue &Offset, bool &isInc,
15208 bool isSEXTLoad, bool isLE, SDValue &Base,
15209 SDValue &Offset, bool &isInc,
15257 ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
15258 SDValue &Offset,
15265 SDValue Ptr;
15306 SDValue &Base,
15307 SDValue &Offset,
15311 SDValue Ptr;
15378 void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
15393 SDValue LHS = Op.getOperand(0);
15394 SDValue RHS = Op.getOperand(1);
15443 const SDValue &SrcSV = Op.getOperand(0);
15472 ARMTargetLowering::targetShrinkDemandedConstant(SDValue Op,
15521 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
15522 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
15751 void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
15753 std::vector<SDValue>&Ops,
15755 SDValue Result;
15957 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
15957 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
15980 const SDValue Dividend = Op->getOperand(0);
15981 const SDValue Divisor = Op->getOperand(1);
15982 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor);
15983 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor);
15984 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
15986 SDValue Values[2] = {Div, Rem};
15992 SDValue InChain = DAG.getEntryNode();
15998 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
16011 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
16011 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
16017 SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
16037 SDValue InChain = DAG.getEntryNode();
16041 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
16052 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
16052 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
16060 SDValue
16061 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
16066 SDValue Chain = Op.getOperand(0);
16067 SDValue Size = Op.getOperand(1);
16072 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
16079 SDValue Ops[2] = { SP, Chain };
16083 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
16086 SDValue Flag;
16093 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
16096 SDValue Ops[2] = { NewSP, Chain };
16100 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
16100 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
16101 SDValue SrcVal = Op.getOperand(0);
16147 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
16147 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
16148 SDValue SrcVal = Op.getOperand(0);
16174 void ARMTargetLowering::lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
16179 SDValue Hi, Lo, Tmp;
lib/Target/ARM/ARMISelLowering.h 304 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
304 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
308 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
331 SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const;
332 SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const;
333 SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const;
334 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
354 bool isZExtFree(SDValue Val, EVT VT2) const override;
360 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
399 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
399 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
406 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
407 SDValue &Offset, ISD::MemIndexedMode &AM,
410 void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
415 bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
438 void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
439 std::vector<SDValue> &Ops,
657 std::pair<SDValue, SDValue> getARMXALUOOp(SDValue Op, SelectionDAG &DAG, SDValue &ARMcc) const;
657 std::pair<SDValue, SDValue> getARMXALUOOp(SDValue Op, SelectionDAG &DAG, SDValue &ARMcc) const;
657 std::pair<SDValue, SDValue> getARMXALUOOp(SDValue Op, SelectionDAG &DAG, SDValue &ARMcc) const;
657 std::pair<SDValue, SDValue> getARMXALUOOp(SDValue Op, SelectionDAG &DAG, SDValue &ARMcc) const;
659 using RegsToPassVector = SmallVector<std::pair<unsigned, SDValue>, 8>;
661 void PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, SDValue Chain,
662 SDValue &Arg, RegsToPassVector &RegsToPass,
664 SDValue &StackPtr,
665 SmallVectorImpl<SDValue> &MemOpChains,
667 SDValue GetF64FormalArgument(CCValAssign &VA, CCValAssign &NextVA,
668 SDValue &Root, SelectionDAG &DAG,
675 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
675 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
675 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
675 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
679 SDValue LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
679 SDValue LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
680 SDValue LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
680 SDValue LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
681 SDValue LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
681 SDValue LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
682 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG,
682 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG,
684 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
684 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
686 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
686 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
687 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
687 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
688 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
688 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
689 SDValue LowerGlobalAddressDarwin(SDValue Op, SelectionDAG &DAG) const;
689 SDValue LowerGlobalAddressDarwin(SDValue Op, SelectionDAG &DAG) const;
690 SDValue LowerGlobalAddressELF(SDValue Op, SelectionDAG &DAG) const;
690 SDValue LowerGlobalAddressELF(SDValue Op, SelectionDAG &DAG) const;
691 SDValue LowerGlobalAddressWindows(SDValue Op, SelectionDAG &DAG) const;
691 SDValue LowerGlobalAddressWindows(SDValue Op, SelectionDAG &DAG) const;
692 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
692 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
693 SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
695 SDValue LowerToTLSExecModels(GlobalAddressSDNode *GA,
698 SDValue LowerGlobalTLSAddressDarwin(SDValue Op, SelectionDAG &DAG) const;
698 SDValue LowerGlobalTLSAddressDarwin(SDValue Op, SelectionDAG &DAG) const;
699 SDValue LowerGlobalTLSAddressWindows(SDValue Op, SelectionDAG &DAG) const;
699 SDValue LowerGlobalTLSAddressWindows(SDValue Op, SelectionDAG &DAG) const;
700 SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const;
700 SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const;
701 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
701 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
702 SDValue LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const;
702 SDValue LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const;
703 SDValue LowerUnsignedALUO(SDValue Op, SelectionDAG &DAG) const;
703 SDValue LowerUnsignedALUO(SDValue Op, SelectionDAG &DAG) const;
704 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
704 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
705 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
705 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
706 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
706 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
707 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
707 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
708 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
708 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
709 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
709 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
710 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
710 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
711 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
711 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
712 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
712 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
713 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
713 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
714 SDValue LowerConstantFP(SDValue Op, SelectionDAG &DAG,
714 SDValue LowerConstantFP(SDValue Op, SelectionDAG &DAG,
716 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
716 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
718 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
718 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
719 SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
719 SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
720 SDValue LowerDivRem(SDValue Op, SelectionDAG &DAG) const;
720 SDValue LowerDivRem(SDValue Op, SelectionDAG &DAG) const;
721 SDValue LowerDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed) const;
721 SDValue LowerDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed) const;
722 void ExpandDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed,
723 SmallVectorImpl<SDValue> &Results) const;
724 SDValue LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG, bool Signed,
724 SDValue LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG, bool Signed,
725 SDValue &Chain) const;
726 SDValue LowerREM(SDNode *N, SelectionDAG &DAG) const;
727 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
727 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
728 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
728 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
729 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
729 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
730 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
730 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
731 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
731 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
732 void lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
738 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
752 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
752 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
754 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
754 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
754 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
758 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
759 SDValue ThisVal) const;
771 SDValue
772 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
775 SmallVectorImpl<SDValue> &InVals) const override;
778 SDValue &Chain, const Value *OrigArg,
783 const SDLoc &dl, SDValue &Chain,
787 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
788 SmallVectorImpl<SDValue> &InVals) const override;
797 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
800 const SmallVectorImpl<SDValue> &OutVals,
809 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
809 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
811 const SmallVectorImpl<SDValue> &OutVals,
814 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
822 SDValue getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, SDValue TrueVal,
822 SDValue getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, SDValue TrueVal,
822 SDValue getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, SDValue TrueVal,
823 SDValue ARMcc, SDValue CCR, SDValue Cmp,
823 SDValue ARMcc, SDValue CCR, SDValue Cmp,
823 SDValue ARMcc, SDValue CCR, SDValue Cmp,
825 SDValue getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
825 SDValue getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
825 SDValue getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
826 SDValue &ARMcc, SelectionDAG &DAG, const SDLoc &dl) const;
827 SDValue getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
827 SDValue getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
827 SDValue getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
829 SDValue duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const;
829 SDValue duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const;
831 SDValue OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const;
831 SDValue OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const;
lib/Target/ARM/ARMSelectionDAGInfo.cpp 23 SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(
24 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
24 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
24 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
25 SDValue Size, unsigned Align, RTLIB::Libcall LC) const {
122 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
122 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
127 SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(
128 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
128 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
128 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
129 SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
156 SDValue TFOps[6];
157 SDValue Loads[6];
241 SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemmove(
242 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
242 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
242 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
243 SDValue Size, unsigned Align, bool isVolatile,
249 SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemset(
250 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
250 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
250 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
251 SDValue Size, unsigned Align, bool isVolatile,
lib/Target/ARM/ARMSelectionDAGInfo.h 40 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
41 SDValue Chain, SDValue Dst, SDValue Src,
41 SDValue Chain, SDValue Dst, SDValue Src,
41 SDValue Chain, SDValue Dst, SDValue Src,
42 SDValue Size, unsigned Align, bool isVolatile,
47 SDValue
48 EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
49 SDValue Dst, SDValue Src, SDValue Size,
49 SDValue Dst, SDValue Src, SDValue Size,
49 SDValue Dst, SDValue Src, SDValue Size,
55 SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
56 SDValue Chain, SDValue Op1, SDValue Op2,
56 SDValue Chain, SDValue Op1, SDValue Op2,
56 SDValue Chain, SDValue Op1, SDValue Op2,
57 SDValue Op3, unsigned Align, bool isVolatile,
60 SDValue EmitSpecializedLibcall(SelectionDAG &DAG, const SDLoc &dl,
61 SDValue Chain, SDValue Dst, SDValue Src,
61 SDValue Chain, SDValue Dst, SDValue Src,
61 SDValue Chain, SDValue Dst, SDValue Src,
62 SDValue Size, unsigned Align,
lib/Target/AVR/AVRISelDAGToDAG.cpp 38 bool SelectAddr(SDNode *Op, SDValue N, SDValue &Base, SDValue &Disp);
38 bool SelectAddr(SDNode *Op, SDValue N, SDValue &Base, SDValue &Disp);
38 bool SelectAddr(SDNode *Op, SDValue N, SDValue &Base, SDValue &Disp);
43 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintCode,
44 std::vector<SDValue> &OutOps) override;
64 bool AVRDAGToDAGISel::SelectAddr(SDNode *Op, SDValue N, SDValue &Base,
64 bool AVRDAGToDAGISel::SelectAddr(SDNode *Op, SDValue N, SDValue &Base,
65 SDValue &Disp) {
202 bool AVRDAGToDAGISel::SelectInlineAsmMemoryOperand(const SDValue &Op,
204 std::vector<SDValue> &OutOps) {
225 SDValue Base, Disp;
240 SDValue CopyFromRegOp = Op->getOperand(0);
241 SDValue ImmOp = Op->getOperand(1);
263 SDValue Base, Disp;
270 SDValue CopyToReg =
273 SDValue NewCopyFromRegOp =
299 SDValue CopyToReg = CurDAG->getCopyToReg(Op, dl, VReg, Op);
300 SDValue CopyFromReg =
314 SDValue TFI =
327 SDValue BasePtr = ST->getBasePtr();
342 SDValue Chain = ST->getChain();
345 SDValue Offset = CurDAG->getTargetConstant(CST, DL, MVT::i16);
346 SDValue Ops[] = {BasePtr.getOperand(0), Offset, ST->getValue(), Chain};
372 SDValue Chain = LD->getChain();
373 SDValue Ptr = LD->getBasePtr();
381 SDValue RegZ = CurDAG->getRegister(AVR::R31R30, MVT::i16);
417 SDValue InFlag;
418 SDValue Chain = N->getOperand(0);
419 SDValue Callee = N->getOperand(1);
435 SmallVector<SDValue, 8> Ops;
457 SDValue Chain = N->getOperand(0);
458 SDValue JmpAddr = N->getOperand(1);
480 SDValue Lhs = N->getOperand(0);
481 SDValue Rhs = N->getOperand(1);
483 SDValue InChain = CurDAG->getEntryNode();
484 SDValue InGlue = SDValue(Mul, 0);
488 SDValue CopyFromLo =
499 SDValue CopyFromHi =
lib/Target/AVR/AVRISelLowering.cpp 278 SDValue AVRTargetLowering::LowerShifts(SDValue Op, SelectionDAG &DAG) const {
278 SDValue AVRTargetLowering::LowerShifts(SDValue Op, SelectionDAG &DAG) const {
310 SDValue Victim = N->getOperand(0);
339 SDValue AVRTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
339 SDValue AVRTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
368 SDValue InChain = DAG.getEntryNode();
372 for (SDValue const &Value : Op->op_values()) {
380 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
394 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
394 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
398 SDValue AVRTargetLowering::LowerGlobalAddress(SDValue Op,
398 SDValue AVRTargetLowering::LowerGlobalAddress(SDValue Op,
406 SDValue Result =
411 SDValue AVRTargetLowering::LowerBlockAddress(SDValue Op,
411 SDValue AVRTargetLowering::LowerBlockAddress(SDValue Op,
416 SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy(DL));
443 SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
443 SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
443 SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
444 SDValue &AVRcc, SelectionDAG &DAG,
446 SDValue Cmp;
538 SDValue LHSlo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, LHS,
540 SDValue LHShi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, LHS,
542 SDValue RHSlo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, RHS,
544 SDValue RHShi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, RHS,
549 SDValue Top = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i8, LHShi,
557 SDValue LHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, LHS,
559 SDValue LHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, LHS,
562 SDValue LHS0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, LHS_0,
564 SDValue LHS1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, LHS_0,
566 SDValue LHS2 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, LHS_1,
568 SDValue LHS3 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, LHS_1,
571 SDValue RHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, RHS,
573 SDValue RHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, RHS,
576 SDValue RHS0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, RHS_0,
578 SDValue RHS1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, RHS_0,
580 SDValue RHS2 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, RHS_1,
582 SDValue RHS3 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i16, RHS_1,
587 SDValue Top = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i8, LHS3,
619 SDValue AVRTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
619 SDValue AVRTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
620 SDValue Chain = Op.getOperand(0);
622 SDValue LHS = Op.getOperand(2);
623 SDValue RHS = Op.getOperand(3);
624 SDValue Dest = Op.getOperand(4);
627 SDValue TargetCC;
628 SDValue Cmp = getAVRCmp(LHS, RHS, CC, TargetCC, DAG, dl);
634 SDValue AVRTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
634 SDValue AVRTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
635 SDValue LHS = Op.getOperand(0);
636 SDValue RHS = Op.getOperand(1);
637 SDValue TrueV = Op.getOperand(2);
638 SDValue FalseV = Op.getOperand(3);
642 SDValue TargetCC;
643 SDValue Cmp = getAVRCmp(LHS, RHS, CC, TargetCC, DAG, dl);
646 SDValue Ops[] = {TrueV, FalseV, TargetCC, Cmp};
651 SDValue AVRTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
651 SDValue AVRTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
652 SDValue LHS = Op.getOperand(0);
653 SDValue RHS = Op.getOperand(1);
657 SDValue TargetCC;
658 SDValue Cmp = getAVRCmp(LHS, RHS, CC, TargetCC, DAG, DL);
660 SDValue TrueV = DAG.getConstant(1, DL, Op.getValueType());
661 SDValue FalseV = DAG.getConstant(0, DL, Op.getValueType());
663 SDValue Ops[] = {TrueV, FalseV, TargetCC, Cmp};
668 SDValue AVRTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
668 SDValue AVRTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
677 SDValue FI = DAG.getFrameIndex(AFI->getVarArgsFrameIndex(), getPointerTy(DL));
683 SDValue AVRTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
683 SDValue AVRTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
716 SmallVectorImpl<SDValue> &Results,
724 SDValue Sub = DAG.getNode(
732 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
772 bool AVRTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
773 SDValue &Offset,
829 SDValue &Base,
830 SDValue &Offset,
915 SDValue Callee = CLI.Callee;
1049 SDValue AVRTargetLowering::LowerFormalArguments(
1050 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1052 SmallVectorImpl<SDValue> &InVals) const {
1065 SDValue ArgValue;
1122 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DL));
1145 SDValue AVRTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1146 SmallVectorImpl<SDValue> &InVals) const {
1150 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1152 SDValue Chain = CLI.Chain;
1153 SDValue Callee = CLI.Callee;
1192 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
1200 SDValue Arg = OutVals[AI];
1242 SDValue Arg = OutVals[Loc];
1247 SDValue PtrOff = DAG.getNode(
1262 SDValue InFlag;
1270 SmallVector<SDValue, 8> Ops;
1311 SDValue AVRTargetLowering::LowerCallResult(
1312 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
1312 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
1314 SmallVectorImpl<SDValue> &InVals) const {
1369 SDValue
1370 AVRTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1373 const SmallVectorImpl<SDValue> &OutVals,
1397 SDValue Flag;
1398 SmallVector<SDValue, 4> RetOps(1, Chain);
1902 void AVRTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
1904 std::vector<SDValue> &Ops,
1906 SDValue Result(0, 0);
lib/Target/AVR/AVRISelLowering.h 86 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
86 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
88 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
95 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
95 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
99 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
100 SDValue &Offset, ISD::MemIndexedMode &AM,
124 void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
125 std::vector<SDValue> &Ops,
137 SDValue getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AVRcc,
137 SDValue getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AVRcc,
137 SDValue getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AVRcc,
137 SDValue getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AVRcc,
139 SDValue LowerShifts(SDValue Op, SelectionDAG &DAG) const;
139 SDValue LowerShifts(SDValue Op, SelectionDAG &DAG) const;
140 SDValue LowerDivRem(SDValue Op, SelectionDAG &DAG) const;
140 SDValue LowerDivRem(SDValue Op, SelectionDAG &DAG) const;
141 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
141 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
142 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
142 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
143 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
143 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
144 SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
144 SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
145 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
145 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
146 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
146 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
147 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
147 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
156 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
156 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
158 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
160 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
160 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
164 SmallVectorImpl<SDValue> &InVals) const override;
165 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
166 SmallVectorImpl<SDValue> &InVals) const override;
167 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
167 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
167 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
171 SmallVectorImpl<SDValue> &InVals) const;
lib/Target/BPF/BPFISelDAGToDAG.cpp 62 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintCode,
63 std::vector<SDValue> &OutOps) override;
73 bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset);
73 bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset);
73 bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset);
74 bool SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset);
74 bool SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset);
74 bool SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset);
99 bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
99 bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
99 bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
135 bool BPFDAGToDAGISel::SelectFIAddr(SDValue Addr, SDValue &Base,
135 bool BPFDAGToDAGISel::SelectFIAddr(SDValue Addr, SDValue &Base,
136 SDValue &Offset) {
160 const SDValue &Op, unsigned ConstraintCode, std::vector<SDValue> &OutOps) {
160 const SDValue &Op, unsigned ConstraintCode, std::vector<SDValue> &OutOps) {
161 SDValue Op0, Op1;
172 SDValue AluOp = CurDAG->getTargetConstant(ISD::ADD, DL, MVT::i32);;
211 SDValue Chain = Node->getOperand(0);
212 SDValue N1 = Node->getOperand(1);
213 SDValue Skb = Node->getOperand(2);
214 SDValue N3 = Node->getOperand(3);
216 SDValue R6Reg = CurDAG->getRegister(BPF::R6, MVT::i64);
228 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
263 SDValue OP1 = LDAddrNode->getOperand(0);
264 SDValue OP2 = LDAddrNode->getOperand(1);
283 SDValue OP1 = LDAddrNode->getOperand(0);
306 SDValue NVal = CurDAG->getConstant(val, DL, MVT::i64);
311 SDValue From[] = {SDValue(Node, 0), SDValue(Node, 1)};
312 SDValue To[] = {NVal, NVal};
477 SDValue BaseV = Node->getOperand(0);
lib/Target/BPF/BPFISelLowering.cpp 46 SDValue Val) {
190 SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
190 SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
206 SDValue BPFTargetLowering::LowerFormalArguments(
207 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
209 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
242 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
275 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
276 SmallVectorImpl<SDValue> &InVals) const {
281 SDValue Chain = CLI.Chain;
282 SDValue Callee = CLI.Callee;
321 SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
328 SDValue Arg = OutVals[i];
354 SDValue InFlag;
379 SmallVector<SDValue, 8> Ops;
406 SDValue
407 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
410 const SmallVectorImpl<SDValue> &OutVals,
429 SDValue Flag;
430 SmallVector<SDValue, 4> RetOps(1, Chain);
454 SDValue BPFTargetLowering::LowerCallResult(
455 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
455 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
457 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
484 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
484 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
498 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
498 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
499 SDValue Chain = Op.getOperand(0);
501 SDValue LHS = Op.getOperand(2);
502 SDValue RHS = Op.getOperand(3);
503 SDValue Dest = Op.getOperand(4);
513 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
513 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
514 SDValue LHS = Op.getOperand(0);
515 SDValue RHS = Op.getOperand(1);
516 SDValue TrueV = Op.getOperand(2);
517 SDValue FalseV = Op.getOperand(3);
524 SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
526 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
551 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
551 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
558 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
lib/Target/BPF/BPFISelLowering.h 40 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
40 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
72 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
72 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
73 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
73 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
74 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
74 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
77 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
77 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
77 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
81 SmallVectorImpl<SDValue> &InVals) const;
87 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
88 SmallVectorImpl<SDValue> &InVals) const override;
91 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
91 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
95 SmallVectorImpl<SDValue> &InVals) const override;
97 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
97 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
99 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
lib/Target/BPF/BPFSelectionDAGInfo.cpp 20 SDValue BPFSelectionDAGInfo::EmitTargetCodeForMemcpy(
21 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
21 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
21 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
22 SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
lib/Target/BPF/BPFSelectionDAGInfo.h 22 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
23 SDValue Chain, SDValue Dst, SDValue Src,
23 SDValue Chain, SDValue Dst, SDValue Src,
23 SDValue Chain, SDValue Dst, SDValue Src,
24 SDValue Size, unsigned Align, bool isVolatile,
lib/Target/Hexagon/HexagonISelDAGToDAG.cpp 67 SDValue Chain = LD->getChain();
68 SDValue Base = LD->getBasePtr();
69 SDValue Offset = LD->getOffset();
128 SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32);
134 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
145 SDValue From[3] = { SDValue(LD,0), SDValue(LD,1), SDValue(LD,2) };
146 SDValue To[3];
168 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
206 SDValue I = CurDAG->getTargetConstant(Inc->getSExtValue(), dl, MVT::i32);
228 SDValue TS;
229 SDValue Loc = IntN->getOperand(3);
265 SDValue Ch = N->getOperand(0);
266 SDValue Loc = N->getOperand(1);
305 SDValue F[] = { SDValue(N,0), SDValue(N,1), SDValue(C,0), SDValue(C,1) };
306 SDValue T[] = { SDValue(L,0), SDValue(S,0), SDValue(L,1), SDValue(S,0) };
365 SmallVector<SDValue, 7> Ops;
391 SDValue I = CurDAG->getTargetConstant(Inc->getSExtValue(), DL, MVT::i32);
426 SDValue I = CurDAG->getTargetConstant(Inc->getSExtValue(), DL, MVT::i32);
464 SDValue Chain = ST->getChain();
465 SDValue Base = ST->getBasePtr();
466 SDValue Offset = ST->getOffset();
467 SDValue Value = ST->getValue();
524 SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32);
528 SDValue From[2] = { SDValue(ST,0), SDValue(ST,1) };
529 SDValue To[2];
533 SDValue Ops[] = { Base, IncV, Value, Chain };
540 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
541 SDValue Ops[] = { Base, Zero, Value, Chain };
570 SDValue Shl_0 = N->getOperand(0);
571 SDValue Shl_1 = N->getOperand(1);
582 SDValue Mul_0 = Shl_0.getOperand(0); // Val
583 SDValue Mul_1 = Shl_0.getOperand(1); // Const
588 SDValue Val = CurDAG->getTargetConstant(ValConst, dl, MVT::i32);
599 SDValue Sub_0 = Shl_0.getOperand(0); // Const 0
600 SDValue Sub_1 = Shl_0.getOperand(1); // Val
604 SDValue Shl2_0 = Sub_1.getOperand(0); // Val
605 SDValue Shl2_1 = Sub_1.getOperand(1); // Const
609 SDValue Val = CurDAG->getTargetConstant(-ValConst, dl, MVT::i32);
683 SDValue V = N->getOperand(1);
684 SDValue U;
686 SDValue R = CurDAG->getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
703 SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i32);
708 SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i64);
738 SDValue FI = CurDAG->getTargetFrameIndex(FX, MVT::i32);
740 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
753 SDValue CH = CurDAG->getEntryNode();
754 SDValue Ops[] = { CurDAG->getCopyFromReg(CH, DL, AR, MVT::i32), FI, Zero };
778 SDValue Ops[] = {
789 SDValue M0 = CurDAG->getTargetConstant(0x18, dl, MVT::i32);
790 SDValue M1 = CurDAG->getTargetConstant(0x03, dl, MVT::i32);
795 SDValue E = CurDAG->getTargetExtractSubreg(Hexagon::isub_lo, dl, ResTy,
811 SDValue A = N->getOperand(1);
815 SDValue M = CurDAG->getTargetConstant(Mask, dl, MVT::i32);
825 SDValue Op = N->getOperand(0);
842 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
855 SDValue C = CurDAG->getTargetConstant(-1, dl, MVT::i32);
868 SDValue C = CurDAG->getTargetConstant(-1, dl, MVT::i32);
911 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
912 std::vector<SDValue> &OutOps) {
913 SDValue Inp = Op, Res;
952 SDValue S0 = U->getOperand(0);
953 SDValue S1 = U->getOperand(1);
954 SDValue SY = (S0.getNode() == I) ? S1 : S0;
966 SDValue LDBasePtr = cast<MemSDNode>(SYNode)->getBasePtr();
967 SDValue STBasePtr = cast<MemSDNode>(UUse)->getBasePtr();
995 SDValue N0 = I->getOperand(0), N1 = I->getOperand(1);
998 SDValue SOp = SelN0 ? N0 : N1;
999 SDValue VOp = SelN0 ? N1 : N0;
1002 SDValue SC = SOp.getOperand(0);
1003 SDValue SX = SOp.getOperand(1);
1004 SDValue SY = SOp.getOperand(2);
1007 SDValue NewOr = DAG.getNode(ISD::OR, DLS, VT, SX, VOp);
1008 SDValue NewSel = DAG.getNode(ISD::SELECT, DLS, VT, SC, NewOr, VOp);
1011 SDValue NewOr = DAG.getNode(ISD::OR, DLS, VT, SY, VOp);
1012 SDValue NewSel = DAG.getNode(ISD::SELECT, DLS, VT, SC, VOp, NewOr);
1033 SDValue Off = I->getOperand(2);
1038 SDValue T0 = Off.getOperand(1);
1043 SDValue T1 = T0.getOperand(0);
1044 SDValue T2 = T0.getOperand(1);
1048 SDValue C = T1.getOperand(1);
1067 SDValue D = DAG.getConstant(DV, DL, VT);
1069 SDValue NewAdd = DAG.getNode(ISD::ADD, DL, VT, T1.getOperand(0), D);
1071 SDValue NewShl = DAG.getNode(ISD::SHL, DL, VT, NewAdd, C);
1096 SDValue Addr = Opc == ISD::LOAD ? N->getOperand(1) : N->getOperand(2);
1100 SDValue T0 = Addr.getOperand(1);
1108 SDValue S = T0.getOperand(0);
1140 SDValue Y = S.getOperand(0);
1144 SDValue D = DAG.getConstant(TZ+CV, dl, VT);
1145 SDValue DC = DAG.getConstant(TZ, dl, VT);
1146 SDValue NewSrl = DAG.getNode(ISD::SRL, dl, VT, Y, D);
1147 SDValue NewShl = DAG.getNode(ISD::SHL, dl, VT, NewSrl, DC);
1161 SDValue OpI1 = N->getOperand(0);
1177 SmallVector<SDValue,2> Ops(U->getNumOperands());
1183 SDValue C0 = DAG.getConstant(0, dl, BVT);
1184 SDValue C1 = DAG.getConstant(1, dl, BVT);
1185 SDValue If0, If1;
1200 SDValue Sel = DAG.getNode(ISD::SELECT, dl, UVT, OpI1, If1, If0);
1279 bool HexagonDAGToDAGISel::SelectAddrFI(SDValue &N, SDValue &R) {
1279 bool HexagonDAGToDAGISel::SelectAddrFI(SDValue &N, SDValue &R) {
1291 inline bool HexagonDAGToDAGISel::SelectAddrGA(SDValue &N, SDValue &R) {
1291 inline bool HexagonDAGToDAGISel::SelectAddrGA(SDValue &N, SDValue &R) {
1295 inline bool HexagonDAGToDAGISel::SelectAddrGP(SDValue &N, SDValue &R) {
1295 inline bool HexagonDAGToDAGISel::SelectAddrGP(SDValue &N, SDValue &R) {
1299 inline bool HexagonDAGToDAGISel::SelectAnyImm(SDValue &N, SDValue &R) {
1299 inline bool HexagonDAGToDAGISel::SelectAnyImm(SDValue &N, SDValue &R) {
1303 inline bool HexagonDAGToDAGISel::SelectAnyImm0(SDValue &N, SDValue &R) {
1303 inline bool HexagonDAGToDAGISel::SelectAnyImm0(SDValue &N, SDValue &R) {
1306 inline bool HexagonDAGToDAGISel::SelectAnyImm1(SDValue &N, SDValue &R) {
1306 inline bool HexagonDAGToDAGISel::SelectAnyImm1(SDValue &N, SDValue &R) {
1309 inline bool HexagonDAGToDAGISel::SelectAnyImm2(SDValue &N, SDValue &R) {
1309 inline bool HexagonDAGToDAGISel::SelectAnyImm2(SDValue &N, SDValue &R) {
1312 inline bool HexagonDAGToDAGISel::SelectAnyImm3(SDValue &N, SDValue &R) {
1312 inline bool HexagonDAGToDAGISel::SelectAnyImm3(SDValue &N, SDValue &R) {
1316 inline bool HexagonDAGToDAGISel::SelectAnyInt(SDValue &N, SDValue &R) {
1316 inline bool HexagonDAGToDAGISel::SelectAnyInt(SDValue &N, SDValue &R) {
1324 bool HexagonDAGToDAGISel::SelectAnyImmediate(SDValue &N, SDValue &R,
1324 bool HexagonDAGToDAGISel::SelectAnyImmediate(SDValue &N, SDValue &R,
1368 bool HexagonDAGToDAGISel::SelectGlobalAddress(SDValue &N, SDValue &R,
1368 bool HexagonDAGToDAGISel::SelectGlobalAddress(SDValue &N, SDValue &R,
1376 SDValue N0 = N.getOperand(0);
1377 SDValue N1 = N.getOperand(1);
1384 SDValue Addr = N0.getOperand(0);
1418 bool HexagonDAGToDAGISel::DetectUseSxtw(SDValue &N, SDValue &R) {
1418 bool HexagonDAGToDAGISel::DetectUseSxtw(SDValue &N, SDValue &R) {
1484 SDValue Ops[] = {
1495 bool HexagonDAGToDAGISel::keepsLowBits(const SDValue &Val, unsigned NumBits,
1496 SDValue &Src) {
1502 const SDValue &Op0 = Val.getOperand(0);
1634 SDValue Value;
1640 WeightedLeaf(SDValue Value, int Weight, int InsertionOrder) :
1731 const SDValue &Val = L.Value;
1758 const SDValue &Val = L.Value;
1779 SDValue HexagonDAGToDAGISel::getMultiplierForSHL(SDNode *N) {
1786 static unsigned getPowerOf2Factor(SDValue Val) {
1809 static bool willShiftRightEliminate(SDValue V, unsigned Amount) {
1811 SDValue Ops[] = { V.getOperand(0), V.getOperand(1) };
1825 SDValue HexagonDAGToDAGISel::factorOutPowerOf2(SDValue V, unsigned Power) {
1825 SDValue HexagonDAGToDAGISel::factorOutPowerOf2(SDValue V, unsigned Power) {
1826 SDValue Ops[] = { V.getOperand(0), V.getOperand(1) };
1850 static bool isTargetConstant(const SDValue &V) {
1877 SDValue HexagonDAGToDAGISel::balanceSubTree(SDNode *N, bool TopLevel) {
1888 SDValue Op0 = N->getOperand(0);
1889 SDValue Op1 = N->getOperand(1);
1926 SmallVector<SDValue, 4> Worklist;
1951 SmallDenseMap<SDValue, int> NodeHeights;
1955 SDValue Child = Worklist.pop_back_val();
2017 SDValue Op1;
2051 SDValue Mul1Factored = factorOutPowerOf2(Mul1.Value, MaxPowerOf2);
2052 SDValue Mul2Factored = factorOutPowerOf2(Mul2.Value, MaxPowerOf2);
2053 SDValue Sum = CurDAG->getNode(ISD::ADD, SDLoc(N), Mul1.Value.getValueType(),
2055 SDValue Const = CurDAG->getConstant(MaxPowerOf2, SDLoc(N),
2057 SDValue New = CurDAG->getNode(ISD::SHL, SDLoc(N), Mul1.Value.getValueType(),
2086 SDValue NewTGA =
2147 SDValue V0 = L0.Value;
2149 SDValue V1 = L1.Value;
2162 SDValue NewNode;
2195 SDValue NewRoot = Leaves.top().Value;
2205 SDValue V0 = NewRoot.getOperand(0);
2238 SDValue BasePtr = cast<MemSDNode>(N)->getBasePtr();
2278 SDValue NewBasePtr = balanceSubTree(BasePtr.getNode(), /*TopLevel=*/ true);
lib/Target/Hexagon/HexagonISelDAGToDAG.h 58 inline bool SelectAddrGA(SDValue &N, SDValue &R);
58 inline bool SelectAddrGA(SDValue &N, SDValue &R);
59 inline bool SelectAddrGP(SDValue &N, SDValue &R);
59 inline bool SelectAddrGP(SDValue &N, SDValue &R);
60 inline bool SelectAnyImm(SDValue &N, SDValue &R);
60 inline bool SelectAnyImm(SDValue &N, SDValue &R);
61 inline bool SelectAnyInt(SDValue &N, SDValue &R);
61 inline bool SelectAnyInt(SDValue &N, SDValue &R);
62 bool SelectAnyImmediate(SDValue &N, SDValue &R, uint32_t LogAlign);
62 bool SelectAnyImmediate(SDValue &N, SDValue &R, uint32_t LogAlign);
63 bool SelectGlobalAddress(SDValue &N, SDValue &R, bool UseGP,
63 bool SelectGlobalAddress(SDValue &N, SDValue &R, bool UseGP,
65 bool SelectAddrFI(SDValue &N, SDValue &R);
65 bool SelectAddrFI(SDValue &N, SDValue &R);
66 bool DetectUseSxtw(SDValue &N, SDValue &R);
66 bool DetectUseSxtw(SDValue &N, SDValue &R);
68 inline bool SelectAnyImm0(SDValue &N, SDValue &R);
68 inline bool SelectAnyImm0(SDValue &N, SDValue &R);
69 inline bool SelectAnyImm1(SDValue &N, SDValue &R);
69 inline bool SelectAnyImm1(SDValue &N, SDValue &R);
70 inline bool SelectAnyImm2(SDValue &N, SDValue &R);
70 inline bool SelectAnyImm2(SDValue &N, SDValue &R);
71 inline bool SelectAnyImm3(SDValue &N, SDValue &R);
71 inline bool SelectAnyImm3(SDValue &N, SDValue &R);
88 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
90 std::vector<SDValue> &OutOps) override;
126 SDValue selectUndef(const SDLoc &dl, MVT ResTy) {
135 bool keepsLowBits(const SDValue &Val, unsigned NumBits, SDValue &Src);
135 bool keepsLowBits(const SDValue &Val, unsigned NumBits, SDValue &Src);
152 SDValue getMultiplierForSHL(SDNode *N);
153 SDValue factorOutPowerOf2(SDValue V, unsigned Power);
153 SDValue factorOutPowerOf2(SDValue V, unsigned Power);
155 SDValue balanceSubTree(SDNode *N, bool Factorize = false);
lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp 618 OpRef(SDValue V) : OpV(V) {}
635 SDValue OpV = SDValue();
832 SDValue getVectorConstant(ArrayRef<uint8_t> Data, const SDLoc &dl);
860 SDValue Va, SDValue Vb, SDNode *N);
860 SDValue Va, SDValue Vb, SDNode *N);
948 SDValue Addr = cast<LoadSDNode>(N)->getBasePtr();
985 std::vector<SDValue> Output;
989 std::vector<SDValue> Ops;
1007 SDValue Op = Output[Idx];
1101 SDValue S = DAG.getTargetConstant(IsRight ? MinSrc : HwLen-MinSrc,
1107 SDValue S = DAG.getTargetConstant(MinSrc, dl, MVT::i32);
1203 SDValue B = getVectorConstant(Bytes, dl);
1361 MVT ResTy, SDValue Va, SDValue Vb,
1361 MVT ResTy, SDValue Va, SDValue Vb,
1388 SmallVector<SDValue,128> Ops;
1396 SDValue Vec;
1412 SDValue Idx = DAG.getConstant(M, dl, MVT::i32);
1413 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LegalTy, {Vec, Idx});
1414 SDValue L = Lower.LowerOperation(Ex, DAG);
1419 SDValue LV;
1421 SDValue B0 = DAG.getBuildVector(SingleTy, dl, {Ops.data(), HwLen});
1422 SDValue L0 = Lower.LowerOperation(B0, DAG);
1423 SDValue B1 = DAG.getBuildVector(SingleTy, dl, {Ops.data()+HwLen, HwLen});
1424 SDValue L1 = Lower.LowerOperation(B1, DAG);
1431 SDValue BV = DAG.getBuildVector(ResTy, dl, Ops);
1456 for (SDValue Op : S->ops()) {
1962 SDValue Ctl = getVectorConstant(FC, dl);
1970 SDValue Ctl = getVectorConstant(RC, dl);
1978 SDValue CtlF = getVectorConstant(FC, dl);
1979 SDValue CtlR = getVectorConstant(RC, dl);
1989 SDValue HvxSelector::getVectorConstant(ArrayRef<uint8_t> Data,
1991 SmallVector<SDValue, 128> Elems;
1995 SDValue BV = DAG.getBuildVector(VecTy, dl, Elems);
1996 SDValue LV = Lower.LowerOperation(BV, DAG);
2047 SDValue Vec0 = N->getOperand(0);
2048 SDValue Vec1 = N->getOperand(1);
2080 SDValue VecV = N->getOperand(0);
2081 SDValue RotV = N->getOperand(1);
2089 SDValue C = DAG.getTargetConstant(S, dl, MVT::i32);
2102 SDValue Vv = N->getOperand(0);
2103 SDValue Vu = N->getOperand(1);
2104 SDValue Rt = N->getOperand(2);
2125 SDValue Chain = N->getOperand(0);
2126 SDValue Address = N->getOperand(2);
2127 SDValue Predicate = N->getOperand(3);
2128 SDValue Base = N->getOperand(4);
2129 SDValue Modifier = N->getOperand(5);
2130 SDValue Offset = N->getOperand(6);
2152 SDValue Ops[] = { Address, Predicate, Base, Modifier, Offset, Chain };
2163 SDValue Chain = N->getOperand(0);
2164 SDValue Address = N->getOperand(2);
2165 SDValue Base = N->getOperand(3);
2166 SDValue Modifier = N->getOperand(4);
2167 SDValue Offset = N->getOperand(5);
2189 SDValue Ops[] = { Address, Base, Modifier, Offset, Chain };
2203 SmallVector<SDValue, 3> Ops = { N->getOperand(1), N->getOperand(2),
2210 SmallVector<SDValue, 3> Ops = { N->getOperand(1), N->getOperand(2),
2217 SmallVector<SDValue, 3> Ops = { N->getOperand(1), N->getOperand(2),
2224 SmallVector<SDValue, 3> Ops = { N->getOperand(1), N->getOperand(2),
lib/Target/Hexagon/HexagonISelLowering.cpp 154 SDValue
155 HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
165 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
165 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
165 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
166 SDValue Chain, ISD::ArgFlagsTy Flags,
168 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
191 SDValue
192 HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
195 const SmallVectorImpl<SDValue> &OutVals,
210 SDValue Flag;
211 SmallVector<SDValue, 4> RetOps(1, Chain);
260 SDValue HexagonTargetLowering::LowerCallResult(
261 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
261 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
263 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
264 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
264 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
278 SDValue RetVal;
286 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
290 SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,
313 SDValue
315 SmallVectorImpl<SDValue> &InVals) const {
319 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
321 SDValue Chain = CLI.Chain;
322 SDValue Callee = CLI.Callee;
370 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
371 SmallVector<SDValue, 8> MemOpChains;
374 SDValue StackPtr =
382 SDValue Arg = OutVals[i];
411 SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
425 SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);
448 SDValue Glue;
499 SmallVector<SDValue, 8> Ops;
546 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
546 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
574 SDValue
575 HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
623 SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
623 SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
625 SDValue Chain = Op.getOperand(0);
626 SDValue Addr = Op.getOperand(1);
630 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
638 SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
638 SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
640 SDValue Chain = Op.getOperand(0);
646 SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
646 SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
648 SDValue Chain = Op.getOperand(0);
652 SDValue Addr = Op.getOperand(2);
654 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
660 SDValue
661 HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
663 SDValue Chain = Op.getOperand(0);
664 SDValue Size = Op.getOperand(1);
665 SDValue Align = Op.getOperand(2);
683 SDValue AC = DAG.getConstant(A, dl, MVT::i32);
685 SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
691 SDValue HexagonTargetLowering::LowerFormalArguments(
692 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
694 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
740 SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
747 SDValue T = DAG.getNode(ISD::AND, dl, RegVT,
772 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
780 SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
798 SDValue
799 HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
804 SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32);
810 SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
810 SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
812 SDValue LHS = Op.getOperand(0);
813 SDValue RHS = Op.getOperand(1);
839 SDValue Op = N.getOperand(0);
869 SDValue
870 HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
871 SDValue PredOp = Op.getOperand(0);
872 SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
916 SDValue
917 HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
935 SDValue T;
952 SDValue
953 HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
957 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
961 SDValue T = DAG.getTargetJumpTable(Idx, VT);
965 SDValue
966 HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
979 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
980 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
991 SDValue
992 HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1000 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1008 SDValue
1009 HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1014 SDValue
1015 HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1026 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
1035 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,
1041 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
1042 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);
1043 SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);
1048 SDValue
1049 HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1056 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);
1060 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL);
1064 SDValue
1065 HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1068 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, PtrVT,
1073 SDValue
1074 HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1075 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1081 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
1096 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),
1110 SDValue
1118 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1125 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,
1128 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1132 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);
1141 SDValue LoadOffset =
1152 SDValue
1160 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1162 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1164 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1174 SDValue
1182 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1186 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);
1189 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1190 SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1193 SDValue InFlag;
1211 SDValue
1212 HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1678 HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl,
1836 bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
1836 bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
1897 std::pair<SDValue, int>
1898 HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
1900 SDValue Op1 = Addr.getOperand(1);
1909 SDValue
1910 HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
1922 SDValue Op0 = Op.getOperand(0);
1923 SDValue Op1 = Op.getOperand(1);
1979 SDValue T0 = DAG.getBitcast(MVT::i32, Op0);
1980 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);
1985 SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl,
1992 SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl,
2006 SDValue T0 = DAG.getBitcast(MVT::i64, Op0);
2007 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);
2036 SDValue
2037 HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2040 if (SDValue S = BVN->getSplatValue()) {
2062 SDValue
2063 HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2067 SDValue
2068 HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2074 SDValue
2075 HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2077 SDValue InpV = Op.getOperand(0);
2084 SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);
2085 SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);
2093 HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2102 SDValue V = Values[i];
2122 SDValue
2123 HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2149 SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32,
2176 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2184 SDValue Vs[4];
2189 SDValue S8 = DAG.getConstant(8, dl, MVT::i32);
2190 SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});
2191 SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});
2192 SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});
2193 SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});
2195 SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);
2205 SDValue
2206 HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2238 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2251 SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);
2257 SDValue L = (ElemTy == MVT::i32)
2260 SDValue H = (ElemTy == MVT::i32)
2266 SDValue
2267 HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2267 HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2295 SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2296 SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);
2297 SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);
2311 SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2313 SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2314 SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);
2332 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2333 SDValue ExtV;
2344 SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);
2353 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2365 SDValue
2366 HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2366 HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2366 HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2373 SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV);
2388 SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32);
2389 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2391 SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2392 SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2412 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2413 SDValue InsV;
2417 SDValue OffV = DAG.getConstant(W, dl, MVT::i32);
2423 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);
2431 SDValue
2432 HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2440 SDValue
2441 HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2449 SDValue
2467 SDValue
2468 HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2472 SmallVector<SDValue,8> Ops;
2484 for (SDValue P : Ops) {
2502 SDValue Rs[8];
2503 SDValue Z = getZero(dl, MVT::i32, DAG);
2507 SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);
2510 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {
2521 SDValue
2522 HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2545 SmallVector<SDValue,4> Words[2];
2548 for (SDValue P : Op.getNode()->op_values()) {
2549 SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);
2560 SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);
2564 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2566 SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2578 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2586 SDValue
2587 HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2589 SDValue Vec = Op.getOperand(0);
2594 SDValue
2595 HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2601 SDValue
2602 HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2608 SDValue
2609 HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
2611 SDValue ValV = Op.getOperand(1);
2629 SDValue
2630 HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
2639 SDValue
2640 HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
2643 SDValue Ptr = SN->getBasePtr();
2654 SDValue
2655 HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
2689 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
2689 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
2701 SDValue Base = LN->getBasePtr();
2702 SDValue Chain = LN->getChain();
2713 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
2717 SDValue Base0 = DAG.getMemBasePlusOffset(BaseNoOff, BO.second, dl);
2718 SDValue Base1 = DAG.getMemBasePlusOffset(BaseNoOff, BO.second+LoadLen, dl);
2729 SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
2730 SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);
2732 SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,
2734 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2736 SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);
2740 SDValue
2741 HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
2742 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
2761 SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});
2762 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),
2767 SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});
2768 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,
2777 SDValue
2778 HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const {
2781 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
2788 SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),
2790 SDValue Out[] = { SubC.getValue(0),
2795 SDValue
2796 HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
2797 SDValue Chain = Op.getOperand(0);
2798 SDValue Offset = Op.getOperand(1);
2799 SDValue Handler = Op.getOperand(2);
2810 SDValue StoreAddr =
2822 SDValue
2823 HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
2832 if (SDValue V = LowerHvxOperation(Op, DAG))
2888 SmallVectorImpl<SDValue> &Results,
2901 SmallVectorImpl<SDValue> &Results,
2912 SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,
2914 SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);
2921 SDValue
2924 SDValue Op(N, 0);
2926 if (SDValue V = PerformHvxDAGCombine(N, DCI))
2935 SDValue P = Op.getOperand(0);
2948 SDValue Cond = Op.getOperand(0);
2950 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
2952 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
2963 SDValue
2964 HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
2968 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
3115 SDValue Callee,
3121 const SmallVectorImpl<SDValue> &OutVals,
3227 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
lib/Target/Hexagon/HexagonISelLowering.h 115 bool IsEligibleForTailCallOptimization(SDValue Callee,
118 const SmallVectorImpl<SDValue> &OutVals,
132 bool hasBitTest(SDValue X, SDValue Y) const override;
132 bool hasBitTest(SDValue X, SDValue Y) const override;
150 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
150 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
151 void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results,
153 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
158 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
158 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
159 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
159 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
160 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
160 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
161 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
161 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
162 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
162 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
163 SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
163 SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
164 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
164 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
165 SDValue LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const;
165 SDValue LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const;
166 SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const;
166 SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const;
167 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
167 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
168 SDValue LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const;
168 SDValue LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const;
169 SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const;
169 SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const;
170 SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const;
170 SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const;
171 SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const;
171 SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const;
172 SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const;
172 SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const;
173 SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const;
173 SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const;
174 SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const;
174 SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const;
175 SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const;
175 SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const;
177 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
177 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
178 SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
178 SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
179 SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
179 SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
180 SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
180 SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
181 SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const;
181 SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const;
182 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
182 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
183 SDValue
184 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
187 SmallVectorImpl<SDValue> &InVals) const override;
188 SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const;
188 SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const;
189 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
189 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
190 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
190 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
191 SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
193 SDValue LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
195 SDValue LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
197 SDValue GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
197 SDValue GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
198 GlobalAddressSDNode *GA, SDValue InFlag, EVT PtrVT,
200 SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const;
200 SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const;
202 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
203 SmallVectorImpl<SDValue> &InVals) const override;
204 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
204 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
204 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
208 SmallVectorImpl<SDValue> &InVals,
209 const SmallVectorImpl<SDValue> &OutVals,
210 SDValue Callee) const;
212 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
212 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
213 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
213 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
214 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
214 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
215 SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const;
215 SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const;
216 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
216 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
223 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
223 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
225 const SmallVectorImpl<SDValue> &OutVals,
228 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
249 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
249 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
250 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
250 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
251 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
251 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
262 SDValue &Base, SDValue &Offset,
262 SDValue &Base, SDValue &Offset,
280 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
280 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
281 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
281 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
312 SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG)
312 SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG)
335 void validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl,
338 std::pair<SDValue,int> getBaseAndOffset(SDValue Addr) const;
338 std::pair<SDValue,int> getBaseAndOffset(SDValue Addr) const;
340 bool getBuildVectorConstInts(ArrayRef<SDValue> Values, MVT VecTy,
343 SDValue buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy,
343 SDValue buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy,
345 SDValue buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy,
345 SDValue buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy,
347 SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl,
347 SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl,
347 SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl,
349 SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
349 SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
349 SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
349 SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
351 SDValue expandPredicate(SDValue Vec32, const SDLoc &dl,
351 SDValue expandPredicate(SDValue Vec32, const SDLoc &dl,
353 SDValue contractPredicate(SDValue Vec64, const SDLoc &dl,
353 SDValue contractPredicate(SDValue Vec64, const SDLoc &dl,
355 SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const;
355 SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const;
357 bool isUndef(SDValue Op) const {
362 SDValue getInstr(unsigned MachineOpc, const SDLoc &dl, MVT Ty,
363 ArrayRef<SDValue> Ops, SelectionDAG &DAG) const {
367 SDValue getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) const;
369 using VectorPair = std::pair<SDValue, SDValue>;
369 using VectorPair = std::pair<SDValue, SDValue>;
372 SDValue getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
372 SDValue getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
375 MVT ty(SDValue Op) const {
401 SDValue opJoin(const VectorPair &Ops, const SDLoc &dl,
403 VectorPair opSplit(SDValue Vec, const SDLoc &dl, SelectionDAG &DAG) const;
404 SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const;
404 SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const;
408 SDValue convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
408 SDValue convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
410 SDValue getIndexInWord32(SDValue Idx, MVT ElemTy, SelectionDAG &DAG) const;
410 SDValue getIndexInWord32(SDValue Idx, MVT ElemTy, SelectionDAG &DAG) const;
411 SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1,
411 SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1,
411 SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1,
414 SDValue buildHvxVectorReg(ArrayRef<SDValue> Values, const SDLoc &dl,
414 SDValue buildHvxVectorReg(ArrayRef<SDValue> Values, const SDLoc &dl,
416 SDValue buildHvxVectorPred(ArrayRef<SDValue> Values, const SDLoc &dl,
416 SDValue buildHvxVectorPred(ArrayRef<SDValue> Values, const SDLoc &dl,
418 SDValue createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
418 SDValue createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
421 SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl,
421 SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl,
421 SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl,
423 SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl,
423 SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl,
423 SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl,
425 SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV,
425 SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV,
425 SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV,
425 SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV,
427 SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV,
427 SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV,
427 SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV,
427 SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV,
429 SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl,
429 SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl,
429 SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl,
431 SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl,
431 SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl,
431 SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl,
433 SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV,
433 SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV,
433 SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV,
433 SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV,
435 SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV,
435 SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV,
435 SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV,
435 SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV,
437 SDValue extendHvxVectorPred(SDValue VecV, const SDLoc &dl, MVT ResTy,
437 SDValue extendHvxVectorPred(SDValue VecV, const SDLoc &dl, MVT ResTy,
440 SDValue LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) const;
440 SDValue LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) const;
441 SDValue LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) const;
441 SDValue LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) const;
442 SDValue LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) const;
442 SDValue LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) const;
443 SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const;
443 SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const;
444 SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const;
444 SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const;
445 SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const;
445 SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const;
447 SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const;
447 SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const;
448 SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const;
448 SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const;
449 SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const;
449 SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const;
450 SDValue LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const;
450 SDValue LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const;
451 SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const;
451 SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const;
452 SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const;
452 SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const;
453 SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const;
453 SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const;
454 SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const;
454 SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const;
455 SDValue LowerHvxShift(SDValue Op, SelectionDAG &DAG) const;
455 SDValue LowerHvxShift(SDValue Op, SelectionDAG &DAG) const;
457 SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const;
457 SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const;
458 SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const;
458 SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const;
464 bool isHvxOperation(SDValue Op) const;
465 SDValue LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const;
465 SDValue LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const;
467 SDValue PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
lib/Target/Hexagon/HexagonISelLoweringHVX.cpp 200 SDValue
201 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
203 SmallVector<SDValue,4> IntOps;
205 for (const SDValue &Op : Ops)
242 SDValue
243 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
251 SDValue
259 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
279 SDValue
280 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
295 SDValue
296 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
306 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
307 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
311 SDValue
312 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
313 SDValue Op1, ArrayRef<int> Mask,
342 SDValue
343 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
354 SmallVector<SDValue,32> Words;
361 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
370 SDValue SplatV;
399 SDValue CP = LowerConstantPool(DAG.getConstantPool(CV, VecTy, Align), DAG);
413 SDValue Vec;
414 for (SDValue V : Values) {
422 SDValue T = V.getOperand(0);
438 SDValue ExtVec;
466 SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
476 SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
477 SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
478 SDValue S = DAG.getConstant(4, dl, MVT::i32);
480 SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
482 SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
490 SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
494 SDValue
495 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
510 SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
521 SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
526 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
528 SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
536 SmallVector<SDValue,4> Words[2];
546 SDValue W0 = isUndef(PredV)
557 for (const SDValue &W : Words[IdxW ^ 1]) {
558 SDValue T = expandPredicate(W, dl, DAG);
563 for (const SDValue &W : Words[IdxW ^ 1]) {
573 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
574 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
575 for (const SDValue &W : Words[IdxW]) {
583 SDValue
584 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
592 SmallVector<SDValue,128> Bytes;
612 for (SDValue V : Values) {
616 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
632 SDValue F = Values[I+B];
636 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
652 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
656 SDValue
657 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
657 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
665 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
666 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
674 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
676 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
680 SDValue
681 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
681 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
688 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
691 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
694 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
695 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
699 SDValue
700 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
700 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
701 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
712 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
714 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
715 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
716 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
718 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
722 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
728 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
730 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
735 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
737 SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
744 SDValue
745 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
745 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
746 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
749 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
752 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
756 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
760 SDValue
761 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
761 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
791 SDValue WordVec = DAG.getBitcast(WordTy, VecV);
794 SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
795 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
799 SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
800 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
801 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
805 SDValue
806 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
806 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
811 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
818 SDValue Undef = DAG.getUNDEF(ByteTy);
832 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
857 SDValue Zero = getZero(dl, MVT::i32, DAG);
858 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
861 SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
862 SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
864 SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
869 SDValue
870 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
870 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
871 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
881 SDValue V0, V1;
882 SDValue SingleV = VecV;
883 SDValue PickHi;
889 SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
902 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
903 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
909 SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
929 SDValue V = DAG.getBitcast(MVT::i32, SubV);
932 SDValue V = DAG.getBitcast(MVT::i64, SubV);
933 SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
934 SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
943 SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
949 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
950 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
956 SDValue
957 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
957 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
958 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
974 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
975 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
976 SDValue ByteIdx;
989 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
994 SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
995 SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
1001 SDValue
1002 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1012 SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
1014 SDValue False = getZero(dl, ResTy, DAG);
1018 SDValue
1019 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1025 SmallVector<SDValue,128> Ops;
1033 ArrayRef<SDValue> A(Ops);
1035 SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
1036 SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
1043 SDValue
1044 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1056 SmallVector<SDValue,8> Elems;
1057 for (SDValue V : Op.getNode()->ops())
1065 SDValue V = Elems[i];
1092 SDValue Op0 = Op.getOperand(0);
1101 SmallVector<SDValue,4> SV(U.begin(), U.end());
1102 ArrayRef<SDValue> Ops(SV);
1105 SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1107 SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1116 SmallVector<SDValue,8> Prefixes;
1117 for (SDValue V : Op.getNode()->op_values()) {
1118 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1124 SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1125 SDValue Res = getZero(dl, ByteTy, DAG);
1133 SDValue
1134 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1137 SDValue VecV = Op.getOperand(0);
1140 SDValue IdxV = Op.getOperand(1);
1147 SDValue
1148 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1151 SDValue VecV = Op.getOperand(0);
1152 SDValue ValV = Op.getOperand(1);
1153 SDValue IdxV = Op.getOperand(2);
1161 SDValue
1162 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1164 SDValue SrcV = Op.getOperand(0);
1167 SDValue IdxV = Op.getOperand(1);
1180 SDValue
1181 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1184 SDValue VecV = Op.getOperand(0);
1185 SDValue ValV = Op.getOperand(1);
1186 SDValue IdxV = Op.getOperand(2);
1197 SDValue
1198 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1204 SDValue InpV = Op.getOperand(0);
1211 SDValue
1212 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1214 SDValue InpV = Op.getOperand(0);
1221 SDValue
1222 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1224 SDValue InpV = Op.getOperand(0);
1231 SDValue
1232 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1237 SDValue InpV = Op.getOperand(0);
1250 SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1252 SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1254 SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1259 SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1266 SDValue
1267 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1275 SDValue Vs = Op.getOperand(0);
1276 SDValue Vt = Op.getOperand(1);
1286 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1294 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1307 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1308 SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1309 SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1310 SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1320 SDValue
1321 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1329 SDValue Vs = Op.getOperand(0);
1330 SDValue Vt = Op.getOperand(1);
1344 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1352 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1357 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1374 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1376 SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1377 SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1380 SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1382 SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1384 SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1398 SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1403 SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1407 SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1410 SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1411 SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1416 SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1419 SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1420 SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1421 SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1422 SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1426 SDValue
1427 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1434 SDValue
1435 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1436 if (SDValue S = getVectorShiftByInt(Op, DAG))
1441 SDValue
1442 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1444 SmallVector<SDValue,2> OpsL, OpsH;
1449 SDValue TV = DAG.getValueType(Ty);
1453 for (SDValue A : Op.getNode()->ops()) {
1468 SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1469 SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1470 SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1474 SDValue
1475 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1485 SDValue Chain = BN->getChain();
1486 SDValue Base0 = BN->getBasePtr();
1487 SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1497 SDValue NewOp;
1500 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1501 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1509 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1510 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1517 SDValue
1518 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1585 SDValue
1589 SDValue Op(N, 0);
1594 SDValue Cond = Op.getOperand(0);
1596 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
1598 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
1608 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp 19 SDValue HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(
20 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
21 SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
61 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
61 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
lib/Target/Hexagon/HexagonSelectionDAGInfo.h 24 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
25 SDValue Chain, SDValue Dst, SDValue Src,
25 SDValue Chain, SDValue Dst, SDValue Src,
25 SDValue Chain, SDValue Dst, SDValue Src,
26 SDValue Size, unsigned Align, bool isVolatile,
lib/Target/Lanai/LanaiISelDAGToDAG.cpp 62 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintCode,
63 std::vector<SDValue> &OutOps) override;
77 bool selectAddrRi(SDValue Addr, SDValue &Base, SDValue &Offset,
77 bool selectAddrRi(SDValue Addr, SDValue &Base, SDValue &Offset,
77 bool selectAddrRi(SDValue Addr, SDValue &Base, SDValue &Offset,
78 SDValue &AluOp);
79 bool selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2, SDValue &AluOp);
79 bool selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2, SDValue &AluOp);
79 bool selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2, SDValue &AluOp);
79 bool selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2, SDValue &AluOp);
80 bool selectAddrSls(SDValue Addr, SDValue &Offset);
80 bool selectAddrSls(SDValue Addr, SDValue &Offset);
81 bool selectAddrSpls(SDValue Addr, SDValue &Base, SDValue &Offset,
81 bool selectAddrSpls(SDValue Addr, SDValue &Base, SDValue &Offset,
81 bool selectAddrSpls(SDValue Addr, SDValue &Base, SDValue &Offset,
82 SDValue &AluOp);
85 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
90 bool selectAddrRiSpls(SDValue Addr, SDValue &Base, SDValue &Offset,
90 bool selectAddrRiSpls(SDValue Addr, SDValue &Base, SDValue &Offset,
90 bool selectAddrRiSpls(SDValue Addr, SDValue &Base, SDValue &Offset,
91 SDValue &AluOp, bool RiMode);
103 bool LanaiDAGToDAGISel::selectAddrSls(SDValue Addr, SDValue &Offset) {
103 bool LanaiDAGToDAGISel::selectAddrSls(SDValue Addr, SDValue &Offset) {
121 bool LanaiDAGToDAGISel::selectAddrRiSpls(SDValue Addr, SDValue &Base,
121 bool LanaiDAGToDAGISel::selectAddrRiSpls(SDValue Addr, SDValue &Base,
122 SDValue &Offset, SDValue &AluOp,
122 SDValue &Offset, SDValue &AluOp,
201 bool LanaiDAGToDAGISel::selectAddrRi(SDValue Addr, SDValue &Base,
201 bool LanaiDAGToDAGISel::selectAddrRi(SDValue Addr, SDValue &Base,
202 SDValue &Offset, SDValue &AluOp) {
202 SDValue &Offset, SDValue &AluOp) {
206 bool LanaiDAGToDAGISel::selectAddrSpls(SDValue Addr, SDValue &Base,
206 bool LanaiDAGToDAGISel::selectAddrSpls(SDValue Addr, SDValue &Base,
207 SDValue &Offset, SDValue &AluOp) {
207 SDValue &Offset, SDValue &AluOp) {
211 bool LanaiDAGToDAGISel::selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2,
211 bool LanaiDAGToDAGISel::selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2,
211 bool LanaiDAGToDAGISel::selectAddrRr(SDValue Addr, SDValue &R1, SDValue &R2,
212 SDValue &AluOp) {
252 const SDValue &Op, unsigned ConstraintCode, std::vector<SDValue> &OutOps) {
252 const SDValue &Op, unsigned ConstraintCode, std::vector<SDValue> &OutOps) {
253 SDValue Op0, Op1, AluOp;
291 SDValue New = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
298 SDValue New = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
317 SDValue Imm = CurDAG->getTargetConstant(0, DL, MVT::i32);
320 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
lib/Target/Lanai/LanaiISelLowering.cpp 175 SDValue LanaiTargetLowering::LowerOperation(SDValue Op,
175 SDValue LanaiTargetLowering::LowerOperation(SDValue Op,
285 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
285 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
287 SDValue Result(nullptr, 0);
396 SDValue LanaiTargetLowering::LowerFormalArguments(
397 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
399 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
409 SDValue LanaiTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
410 SmallVectorImpl<SDValue> &InVals) const {
414 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
416 SDValue Chain = CLI.Chain;
417 SDValue Callee = CLI.Callee;
437 SDValue LanaiTargetLowering::LowerCCCArguments(
438 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
440 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
465 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
503 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
519 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[0]);
533 SDValue
534 LanaiTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
537 const SmallVectorImpl<SDValue> &OutVals,
549 SDValue Flag;
550 SmallVector<SDValue, 4> RetOps(1, Chain);
574 SDValue Val =
596 SDValue LanaiTargetLowering::LowerCCCCallTo(
597 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool IsVarArg,
597 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool IsVarArg,
599 const SmallVectorImpl<SDValue> &OutVals,
601 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
628 SmallVector<SDValue, 8> ByValArgs;
634 SDValue Arg = OutVals[I];
639 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
640 SDValue SizeNode = DAG.getConstant(Size, DL, MVT::i32);
652 SmallVector<std::pair<unsigned, SDValue>, 4> RegsToPass;
653 SmallVector<SDValue, 12> MemOpChains;
654 SDValue StackPtr;
659 SDValue Arg = OutVals[I];
694 SDValue PtrOff =
709 SDValue InFlag;
734 SmallVector<SDValue, 8> Ops;
774 SDValue LanaiTargetLowering::LowerCallResult(
775 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
775 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
777 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
801 static LPCC::CondCode IntCondCCodeToICC(SDValue CC, const SDLoc &DL,
802 SDValue &RHS, SelectionDAG &DAG) {
866 SDValue LanaiTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
866 SDValue LanaiTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
867 SDValue Chain = Op.getOperand(0);
868 SDValue Cond = Op.getOperand(1);
869 SDValue LHS = Op.getOperand(2);
870 SDValue RHS = Op.getOperand(3);
871 SDValue Dest = Op.getOperand(4);
875 SDValue TargetCC = DAG.getConstant(CC, DL, MVT::i32);
876 SDValue Flag =
883 SDValue LanaiTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
883 SDValue LanaiTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
936 SDValue Res;
938 SDValue V = Op->getOperand(0);
959 SDValue Op =
969 SDValue LanaiTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
969 SDValue LanaiTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
970 SDValue LHS = Op.getOperand(0);
971 SDValue RHS = Op.getOperand(1);
972 SDValue Cond = Op.getOperand(2);
976 SDValue TargetCC = DAG.getConstant(CC, DL, MVT::i32);
977 SDValue Flag =
983 SDValue LanaiTargetLowering::LowerSELECT_CC(SDValue Op,
983 SDValue LanaiTargetLowering::LowerSELECT_CC(SDValue Op,
985 SDValue LHS = Op.getOperand(0);
986 SDValue RHS = Op.getOperand(1);
987 SDValue TrueV = Op.getOperand(2);
988 SDValue FalseV = Op.getOperand(3);
989 SDValue Cond = Op.getOperand(4);
993 SDValue TargetCC = DAG.getConstant(CC, DL, MVT::i32);
994 SDValue Flag =
1002 SDValue LanaiTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
1002 SDValue LanaiTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
1007 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
1017 SDValue LanaiTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1017 SDValue LanaiTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1019 SDValue Chain = Op.getOperand(0);
1020 SDValue Size = Op.getOperand(1);
1026 SDValue StackPointer = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i32);
1030 SDValue Sub = DAG.getNode(ISD::SUB, DL, MVT::i32, StackPointer, Size);
1042 SDValue ArgAdjust = DAG.getNode(LanaiISD::ADJDYNALLOC, DL, MVT::i32, Sub);
1046 SDValue CopyChain = DAG.getCopyToReg(Chain, DL, SPReg, Sub);
1048 SDValue Ops[2] = {ArgAdjust, CopyChain};
1052 SDValue LanaiTargetLowering::LowerRETURNADDR(SDValue Op,
1052 SDValue LanaiTargetLowering::LowerRETURNADDR(SDValue Op,
1062 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1064 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
1075 SDValue LanaiTargetLowering::LowerFRAMEADDR(SDValue Op,
1075 SDValue LanaiTargetLowering::LowerFRAMEADDR(SDValue Op,
1082 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, Lanai::FP, VT);
1086 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
1125 SDValue LanaiTargetLowering::LowerConstantPool(SDValue Op,
1125 SDValue LanaiTargetLowering::LowerConstantPool(SDValue Op,
1138 SDValue Small = DAG.getTargetConstantPool(
1147 SDValue Hi = DAG.getTargetConstantPool(C, MVT::i32, N->getAlignment(),
1149 SDValue Lo = DAG.getTargetConstantPool(C, MVT::i32, N->getAlignment(),
1153 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i32, Hi, Lo);
1158 SDValue LanaiTargetLowering::LowerGlobalAddress(SDValue Op,
1158 SDValue LanaiTargetLowering::LowerGlobalAddress(SDValue Op,
1172 SDValue Small = DAG.getTargetGlobalAddress(
1182 SDValue Hi = DAG.getTargetGlobalAddress(
1184 SDValue Lo = DAG.getTargetGlobalAddress(
1192 SDValue LanaiTargetLowering::LowerBlockAddress(SDValue Op,
1192 SDValue LanaiTargetLowering::LowerBlockAddress(SDValue Op,
1200 SDValue Hi = DAG.getBlockAddress(BA, MVT::i32, true, OpFlagHi);
1201 SDValue Lo = DAG.getBlockAddress(BA, MVT::i32, true, OpFlagLo);
1204 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i32, Hi, Lo);
1208 SDValue LanaiTargetLowering::LowerJumpTable(SDValue Op,
1208 SDValue LanaiTargetLowering::LowerJumpTable(SDValue Op,
1215 SDValue Small = DAG.getTargetJumpTable(
1224 SDValue Hi = DAG.getTargetJumpTable(
1226 SDValue Lo = DAG.getTargetJumpTable(
1230 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i32, Hi, Lo);
1235 SDValue LanaiTargetLowering::LowerSHL_PARTS(SDValue Op,
1235 SDValue LanaiTargetLowering::LowerSHL_PARTS(SDValue Op,
1241 SDValue ShOpLo = Op.getOperand(0);
1242 SDValue ShOpHi = Op.getOperand(1);
1243 SDValue ShAmt = Op.getOperand(2);
1252 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
1254 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
1258 SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
1259 SDValue SetCC = DAG.getSetCC(dl, MVT::i32, ShAmt, Zero, ISD::SETEQ);
1262 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
1264 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
1265 SDValue HiForNormalShift =
1268 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
1271 SDValue Hi =
1276 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
1277 SDValue Lo = DAG.getSelect(
1280 SDValue Ops[2] = {Lo, Hi};
1284 SDValue LanaiTargetLowering::LowerSRL_PARTS(SDValue Op,
1284 SDValue LanaiTargetLowering::LowerSRL_PARTS(SDValue Op,
1289 SDValue ShOpLo = Op.getOperand(0);
1290 SDValue ShOpHi = Op.getOperand(1);
1291 SDValue ShAmt = Op.getOperand(2);
1304 SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
1305 SDValue NegatedPlus32 = DAG.getNode(
1307 SDValue SetCC = DAG.getSetCC(dl, MVT::i32, NegatedPlus32, Zero, ISD::SETLE);
1309 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i32, ShOpHi, ShAmt);
1312 SDValue Lo = DAG.getNode(ISD::SRL, dl, MVT::i32, ShOpLo, ShAmt);
1314 SDValue CarryBits =
1316 SDValue ShiftIsZero = DAG.getSetCC(dl, MVT::i32, ShAmt, Zero, ISD::SETEQ);
1320 SDValue Ops[2] = {Lo, Hi};
1325 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
1347 static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC,
1348 bool &Invert, SDValue &OtherOp,
1355 SDValue N1 = N->getOperand(1);
1356 SDValue N2 = N->getOperand(2);
1418 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
1418 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
1418 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
1423 SDValue NonConstantVal;
1424 SDValue CCOp;
1431 SDValue TrueVal = OtherOp;
1432 SDValue FalseVal =
1442 static SDValue
1445 SDValue N0 = N->getOperand(0);
1446 SDValue N1 = N->getOperand(1);
1448 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
1451 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
1457 static SDValue PerformSUBCombine(SDNode *N,
1459 SDValue N0 = N->getOperand(0);
1460 SDValue N1 = N->getOperand(1);
1464 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, /*AllOnes=*/false))
1470 SDValue LanaiTargetLowering::PerformDAGCombine(SDNode *N,
1489 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
lib/Target/Lanai/LanaiISelLowering.h 72 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
72 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
78 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
78 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
79 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
79 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
80 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
80 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
81 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
81 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
82 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
82 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
83 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
83 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
84 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
84 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
101 void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
102 std::vector<SDValue> &Ops,
105 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
107 void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
113 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
113 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
113 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
117 const SmallVectorImpl<SDValue> &OutVals,
120 SmallVectorImpl<SDValue> &InVals) const;
122 SDValue LowerCCCArguments(SDValue Chain, CallingConv::ID CallConv,
122 SDValue LowerCCCArguments(SDValue Chain, CallingConv::ID CallConv,
126 SmallVectorImpl<SDValue> &InVals) const;
128 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
128 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
128 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
132 SmallVectorImpl<SDValue> &InVals) const;
134 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
135 SmallVectorImpl<SDValue> &InVals) const override;
137 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
137 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
141 SmallVectorImpl<SDValue> &InVals) const override;
143 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
143 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
145 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
lib/Target/Lanai/LanaiSelectionDAGInfo.cpp 21 SDValue LanaiSelectionDAGInfo::EmitTargetCodeForMemcpy(
22 SelectionDAG & /*DAG*/, const SDLoc & /*dl*/, SDValue /*Chain*/,
23 SDValue /*Dst*/, SDValue /*Src*/, SDValue Size, unsigned /*Align*/,
23 SDValue /*Dst*/, SDValue /*Src*/, SDValue Size, unsigned /*Align*/,
23 SDValue /*Dst*/, SDValue /*Src*/, SDValue Size, unsigned /*Align*/,
lib/Target/Lanai/LanaiSelectionDAGInfo.h 25 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
26 SDValue Chain, SDValue Dst, SDValue Src,
26 SDValue Chain, SDValue Dst, SDValue Src,
26 SDValue Chain, SDValue Dst, SDValue Src,
27 SDValue Size, unsigned Align, bool isVolatile,
lib/Target/MSP430/MSP430ISelDAGToDAG.cpp 43 SDValue Reg;
105 bool MatchAddress(SDValue N, MSP430ISelAddressMode &AM);
106 bool MatchWrapper(SDValue N, MSP430ISelAddressMode &AM);
107 bool MatchAddressBase(SDValue N, MSP430ISelAddressMode &AM);
109 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
110 std::vector<SDValue> &OutOps) override;
119 bool tryIndexedBinOp(SDNode *Op, SDValue N1, SDValue N2, unsigned Opc8,
119 bool tryIndexedBinOp(SDNode *Op, SDValue N1, SDValue N2, unsigned Opc8,
122 bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Disp);
122 bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Disp);
122 bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Disp);
138 bool MSP430DAGToDAGISel::MatchWrapper(SDValue N, MSP430ISelAddressMode &AM) {
144 SDValue N0 = N.getOperand(0);
170 bool MSP430DAGToDAGISel::MatchAddressBase(SDValue N, MSP430ISelAddressMode &AM) {
183 bool MSP430DAGToDAGISel::MatchAddress(SDValue N, MSP430ISelAddressMode &AM) {
247 bool MSP430DAGToDAGISel::SelectAddr(SDValue N,
248 SDValue &Base, SDValue &Disp) {
248 SDValue &Base, SDValue &Disp) {
285 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
286 std::vector<SDValue> &OutOps) {
287 SDValue Op0, Op1;
353 bool MSP430DAGToDAGISel::tryIndexedBinOp(SDNode *Op, SDValue N1, SDValue N2,
353 bool MSP430DAGToDAGISel::tryIndexedBinOp(SDNode *Op, SDValue N1, SDValue N2,
365 SDValue Ops0[] = { N2, LD->getBasePtr(), LD->getChain() };
396 SDValue TFI = CurDAG->getTargetFrameIndex(FI, MVT::i16);
lib/Target/MSP430/MSP430ISelLowering.cpp 334 SDValue MSP430TargetLowering::LowerOperation(SDValue Op,
334 SDValue MSP430TargetLowering::LowerOperation(SDValue Op,
551 SDValue MSP430TargetLowering::LowerFormalArguments(
552 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
554 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
569 SDValue
571 SmallVectorImpl<SDValue> &InVals) const {
575 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
577 SDValue Chain = CLI.Chain;
578 SDValue Callee = CLI.Callee;
602 SDValue MSP430TargetLowering::LowerCCCArguments(
603 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
605 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
640 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
661 SDValue InVal;
681 SDValue FIN = DAG.getFrameIndex(FI, MVT::i16);
699 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[i]);
718 SDValue
719 MSP430TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
722 const SmallVectorImpl<SDValue> &OutVals,
741 SDValue Flag;
742 SmallVector<SDValue, 4> RetOps(1, Chain);
765 SDValue Val =
788 SDValue MSP430TargetLowering::LowerCCCCallTo(
789 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
789 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
791 const SmallVectorImpl<SDValue> &OutVals,
793 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
806 SmallVector<std::pair<unsigned, SDValue>, 4> RegsToPass;
807 SmallVector<SDValue, 12> MemOpChains;
808 SDValue StackPtr;
814 SDValue Arg = OutVals[i];
841 SDValue PtrOff =
845 SDValue MemOp;
849 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i16);
873 SDValue InFlag;
890 SmallVector<SDValue, 8> Ops;
920 SDValue MSP430TargetLowering::LowerCallResult(
921 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
921 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
923 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
943 SDValue MSP430TargetLowering::LowerShifts(SDValue Op,
943 SDValue MSP430TargetLowering::LowerShifts(SDValue Op,
957 SDValue Victim = N->getOperand(0);
996 SDValue MSP430TargetLowering::LowerGlobalAddress(SDValue Op,
996 SDValue MSP430TargetLowering::LowerGlobalAddress(SDValue Op,
1003 SDValue Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op), PtrVT, Offset);
1007 SDValue MSP430TargetLowering::LowerExternalSymbol(SDValue Op,
1007 SDValue MSP430TargetLowering::LowerExternalSymbol(SDValue Op,
1012 SDValue Result = DAG.getTargetExternalSymbol(Sym, PtrVT);
1017 SDValue MSP430TargetLowering::LowerBlockAddress(SDValue Op,
1017 SDValue MSP430TargetLowering::LowerBlockAddress(SDValue Op,
1022 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT);
1027 static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
1027 static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
1027 static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
1027 static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
1113 SDValue MSP430TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
1113 SDValue MSP430TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
1114 SDValue Chain = Op.getOperand(0);
1116 SDValue LHS = Op.getOperand(2);
1117 SDValue RHS = Op.getOperand(3);
1118 SDValue Dest = Op.getOperand(4);
1121 SDValue TargetCC;
1122 SDValue Flag = EmitCMP(LHS, RHS, TargetCC, CC, dl, DAG);
1128 SDValue MSP430TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1128 SDValue MSP430TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1129 SDValue LHS = Op.getOperand(0);
1130 SDValue RHS = Op.getOperand(1);
1148 SDValue TargetCC;
1149 SDValue Flag = EmitCMP(LHS, RHS, TargetCC, CC, dl, DAG);
1185 SDValue One = DAG.getConstant(1, dl, VT);
1187 SDValue SR = DAG.getCopyFromReg(DAG.getEntryNode(), dl, MSP430::SR,
1197 SDValue Zero = DAG.getConstant(0, dl, VT);
1199 SDValue Ops[] = {One, Zero, TargetCC, Flag};
1204 SDValue MSP430TargetLowering::LowerSELECT_CC(SDValue Op,
1204 SDValue MSP430TargetLowering::LowerSELECT_CC(SDValue Op,
1206 SDValue LHS = Op.getOperand(0);
1207 SDValue RHS = Op.getOperand(1);
1208 SDValue TrueV = Op.getOperand(2);
1209 SDValue FalseV = Op.getOperand(3);
1213 SDValue TargetCC;
1214 SDValue Flag = EmitCMP(LHS, RHS, TargetCC, CC, dl, DAG);
1217 SDValue Ops[] = {TrueV, FalseV, TargetCC, Flag};
1222 SDValue MSP430TargetLowering::LowerSIGN_EXTEND(SDValue Op,
1222 SDValue MSP430TargetLowering::LowerSIGN_EXTEND(SDValue Op,
1224 SDValue Val = Op.getOperand(0);
1235 SDValue
1253 SDValue MSP430TargetLowering::LowerRETURNADDR(SDValue Op,
1253 SDValue MSP430TargetLowering::LowerRETURNADDR(SDValue Op,
1266 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1267 SDValue Offset =
1275 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
1280 SDValue MSP430TargetLowering::LowerFRAMEADDR(SDValue Op,
1280 SDValue MSP430TargetLowering::LowerFRAMEADDR(SDValue Op,
1288 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1296 SDValue MSP430TargetLowering::LowerVASTART(SDValue Op,
1296 SDValue MSP430TargetLowering::LowerVASTART(SDValue Op,
1303 SDValue FrameIndex =
1312 SDValue MSP430TargetLowering::LowerJumpTable(SDValue Op,
1312 SDValue MSP430TargetLowering::LowerJumpTable(SDValue Op,
1316 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
1324 SDValue &Base,
1325 SDValue &Offset,
1401 bool MSP430TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
lib/Target/MSP430/MSP430ISelLowering.h 83 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
83 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
89 SDValue LowerShifts(SDValue Op, SelectionDAG &DAG) const;
89 SDValue LowerShifts(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
93 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
93 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
94 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
94 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
95 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
95 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
96 SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const;
96 SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const;
97 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
97 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
98 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
98 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
99 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
99 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
100 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
100 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
101 SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
125 bool isZExtFree(SDValue Val, EVT VT2) const override;
136 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
136 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
136 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
140 const SmallVectorImpl<SDValue> &OutVals,
143 SmallVectorImpl<SDValue> &InVals) const;
145 SDValue LowerCCCArguments(SDValue Chain, CallingConv::ID CallConv,
145 SDValue LowerCCCArguments(SDValue Chain, CallingConv::ID CallConv,
149 SmallVectorImpl<SDValue> &InVals) const;
151 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
151 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
151 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
155 SmallVectorImpl<SDValue> &InVals) const;
157 SDValue
158 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
161 SmallVectorImpl<SDValue> &InVals) const override;
162 SDValue
164 SmallVectorImpl<SDValue> &InVals) const override;
172 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
172 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
174 const SmallVectorImpl<SDValue> &OutVals,
178 SDValue &Base,
179 SDValue &Offset,
lib/Target/Mips/Mips16ISelDAGToDAG.cpp 50 SDValue InFlag = SDValue(Mul, 0);
98 bool Mips16DAGToDAGISel::selectAddr(bool SPAllowed, SDValue Addr, SDValue &Base,
98 bool Mips16DAGToDAGISel::selectAddr(bool SPAllowed, SDValue Addr, SDValue &Base,
99 SDValue &Offset) {
153 SDValue Opnd0 = Addr.getOperand(1).getOperand(0);
167 bool Mips16DAGToDAGISel::selectAddr16(SDValue Addr, SDValue &Base,
167 bool Mips16DAGToDAGISel::selectAddr16(SDValue Addr, SDValue &Base,
168 SDValue &Offset) {
172 bool Mips16DAGToDAGISel::selectAddr16SP(SDValue Addr, SDValue &Base,
172 bool Mips16DAGToDAGISel::selectAddr16SP(SDValue Addr, SDValue &Base,
173 SDValue &Offset) {
lib/Target/Mips/Mips16ISelDAGToDAG.h 32 bool selectAddr(bool SPAllowed, SDValue Addr, SDValue &Base,
32 bool selectAddr(bool SPAllowed, SDValue Addr, SDValue &Base,
33 SDValue &Offset);
34 bool selectAddr16(SDValue Addr, SDValue &Base,
34 bool selectAddr16(SDValue Addr, SDValue &Base,
35 SDValue &Offset) override;
36 bool selectAddr16SP(SDValue Addr, SDValue &Base,
36 bool selectAddr16SP(SDValue Addr, SDValue &Base,
37 SDValue &Offset) override;
lib/Target/Mips/Mips16ISelLowering.cpp 412 getOpndList(SmallVectorImpl<SDValue> &Ops,
413 std::deque< std::pair<unsigned, SDValue> > &RegsToPass,
415 bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
416 SDValue Chain) const {
485 SDValue JumpTarget = Callee;
lib/Target/Mips/Mips16ISelLowering.h 47 getOpndList(SmallVectorImpl<SDValue> &Ops,
48 std::deque< std::pair<unsigned, SDValue> > &RegsToPass,
50 bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
51 SDValue Chain) const override;
lib/Target/Mips/MipsISelDAGToDAG.cpp 76 bool MipsDAGToDAGISel::selectAddrRegImm(SDValue Addr, SDValue &Base,
76 bool MipsDAGToDAGISel::selectAddrRegImm(SDValue Addr, SDValue &Base,
77 SDValue &Offset) const {
82 bool MipsDAGToDAGISel::selectAddrDefault(SDValue Addr, SDValue &Base,
82 bool MipsDAGToDAGISel::selectAddrDefault(SDValue Addr, SDValue &Base,
83 SDValue &Offset) const {
88 bool MipsDAGToDAGISel::selectIntAddr(SDValue Addr, SDValue &Base,
88 bool MipsDAGToDAGISel::selectIntAddr(SDValue Addr, SDValue &Base,
89 SDValue &Offset) const {
94 bool MipsDAGToDAGISel::selectIntAddr11MM(SDValue Addr, SDValue &Base,
94 bool MipsDAGToDAGISel::selectIntAddr11MM(SDValue Addr, SDValue &Base,
95 SDValue &Offset) const {
100 bool MipsDAGToDAGISel::selectIntAddr12MM(SDValue Addr, SDValue &Base,
100 bool MipsDAGToDAGISel::selectIntAddr12MM(SDValue Addr, SDValue &Base,
101 SDValue &Offset) const {
106 bool MipsDAGToDAGISel::selectIntAddr16MM(SDValue Addr, SDValue &Base,
106 bool MipsDAGToDAGISel::selectIntAddr16MM(SDValue Addr, SDValue &Base,
107 SDValue &Offset) const {
112 bool MipsDAGToDAGISel::selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
112 bool MipsDAGToDAGISel::selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
113 SDValue &Offset) const {
118 bool MipsDAGToDAGISel::selectIntAddrSImm10(SDValue Addr, SDValue &Base,
118 bool MipsDAGToDAGISel::selectIntAddrSImm10(SDValue Addr, SDValue &Base,
119 SDValue &Offset) const {
124 bool MipsDAGToDAGISel::selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
124 bool MipsDAGToDAGISel::selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
125 SDValue &Offset) const {
130 bool MipsDAGToDAGISel::selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
130 bool MipsDAGToDAGISel::selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
131 SDValue &Offset) const {
136 bool MipsDAGToDAGISel::selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
136 bool MipsDAGToDAGISel::selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
137 SDValue &Offset) const {
142 bool MipsDAGToDAGISel::selectAddr16(SDValue Addr, SDValue &Base,
142 bool MipsDAGToDAGISel::selectAddr16(SDValue Addr, SDValue &Base,
143 SDValue &Offset) {
148 bool MipsDAGToDAGISel::selectAddr16SP(SDValue Addr, SDValue &Base,
148 bool MipsDAGToDAGISel::selectAddr16SP(SDValue Addr, SDValue &Base,
149 SDValue &Offset) {
160 bool MipsDAGToDAGISel::selectVSplatUimm1(SDValue N, SDValue &Imm) const {
160 bool MipsDAGToDAGISel::selectVSplatUimm1(SDValue N, SDValue &Imm) const {
165 bool MipsDAGToDAGISel::selectVSplatUimm2(SDValue N, SDValue &Imm) const {
165 bool MipsDAGToDAGISel::selectVSplatUimm2(SDValue N, SDValue &Imm) const {
170 bool MipsDAGToDAGISel::selectVSplatUimm3(SDValue N, SDValue &Imm) const {
170 bool MipsDAGToDAGISel::selectVSplatUimm3(SDValue N, SDValue &Imm) const {
175 bool MipsDAGToDAGISel::selectVSplatUimm4(SDValue N, SDValue &Imm) const {
175 bool MipsDAGToDAGISel::selectVSplatUimm4(SDValue N, SDValue &Imm) const {
180 bool MipsDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &Imm) const {
180 bool MipsDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &Imm) const {
185 bool MipsDAGToDAGISel::selectVSplatUimm6(SDValue N, SDValue &Imm) const {
185 bool MipsDAGToDAGISel::selectVSplatUimm6(SDValue N, SDValue &Imm) const {
190 bool MipsDAGToDAGISel::selectVSplatUimm8(SDValue N, SDValue &Imm) const {
190 bool MipsDAGToDAGISel::selectVSplatUimm8(SDValue N, SDValue &Imm) const {
195 bool MipsDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &Imm) const {
195 bool MipsDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &Imm) const {
200 bool MipsDAGToDAGISel::selectVSplatUimmPow2(SDValue N, SDValue &Imm) const {
200 bool MipsDAGToDAGISel::selectVSplatUimmPow2(SDValue N, SDValue &Imm) const {
205 bool MipsDAGToDAGISel::selectVSplatUimmInvPow2(SDValue N, SDValue &Imm) const {
205 bool MipsDAGToDAGISel::selectVSplatUimmInvPow2(SDValue N, SDValue &Imm) const {
210 bool MipsDAGToDAGISel::selectVSplatMaskL(SDValue N, SDValue &Imm) const {
210 bool MipsDAGToDAGISel::selectVSplatMaskL(SDValue N, SDValue &Imm) const {
215 bool MipsDAGToDAGISel::selectVSplatMaskR(SDValue N, SDValue &Imm) const {
215 bool MipsDAGToDAGISel::selectVSplatMaskR(SDValue N, SDValue &Imm) const {
229 SDValue X = Node->getOperand(0);
230 SDValue C = Node->getOperand(1);
255 SDValue NegC = CurDAG->FoldConstantArithmetic(
258 SDValue NewNode = CurDAG->getNode(ISD::SUB, DL, VT, X, NegC);
311 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
312 std::vector<SDValue> &OutOps) {
lib/Target/Mips/MipsISelDAGToDAG.h 58 virtual bool selectAddrRegImm(SDValue Addr, SDValue &Base,
58 virtual bool selectAddrRegImm(SDValue Addr, SDValue &Base,
59 SDValue &Offset) const;
62 virtual bool selectAddrDefault(SDValue Addr, SDValue &Base,
62 virtual bool selectAddrDefault(SDValue Addr, SDValue &Base,
63 SDValue &Offset) const;
66 virtual bool selectIntAddr(SDValue Addr, SDValue &Base,
66 virtual bool selectIntAddr(SDValue Addr, SDValue &Base,
67 SDValue &Offset) const;
69 virtual bool selectIntAddr11MM(SDValue Addr, SDValue &Base,
69 virtual bool selectIntAddr11MM(SDValue Addr, SDValue &Base,
70 SDValue &Offset) const;
72 virtual bool selectIntAddr12MM(SDValue Addr, SDValue &Base,
72 virtual bool selectIntAddr12MM(SDValue Addr, SDValue &Base,
73 SDValue &Offset) const;
75 virtual bool selectIntAddr16MM(SDValue Addr, SDValue &Base,
75 virtual bool selectIntAddr16MM(SDValue Addr, SDValue &Base,
76 SDValue &Offset) const;
78 virtual bool selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
78 virtual bool selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
79 SDValue &Offset) const;
82 virtual bool selectIntAddrSImm10(SDValue Addr, SDValue &Base,
82 virtual bool selectIntAddrSImm10(SDValue Addr, SDValue &Base,
83 SDValue &Offset) const;
85 virtual bool selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
85 virtual bool selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
86 SDValue &Offset) const;
88 virtual bool selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
88 virtual bool selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
89 SDValue &Offset) const;
91 virtual bool selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
91 virtual bool selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
92 SDValue &Offset) const;
94 virtual bool selectAddr16(SDValue Addr, SDValue &Base, SDValue &Offset);
94 virtual bool selectAddr16(SDValue Addr, SDValue &Base, SDValue &Offset);
94 virtual bool selectAddr16(SDValue Addr, SDValue &Base, SDValue &Offset);
95 virtual bool selectAddr16SP(SDValue Addr, SDValue &Base, SDValue &Offset);
95 virtual bool selectAddr16SP(SDValue Addr, SDValue &Base, SDValue &Offset);
95 virtual bool selectAddr16SP(SDValue Addr, SDValue &Base, SDValue &Offset);
101 virtual bool selectVSplatUimm1(SDValue N, SDValue &Imm) const;
101 virtual bool selectVSplatUimm1(SDValue N, SDValue &Imm) const;
103 virtual bool selectVSplatUimm2(SDValue N, SDValue &Imm) const;
103 virtual bool selectVSplatUimm2(SDValue N, SDValue &Imm) const;
105 virtual bool selectVSplatUimm3(SDValue N, SDValue &Imm) const;
105 virtual bool selectVSplatUimm3(SDValue N, SDValue &Imm) const;
107 virtual bool selectVSplatUimm4(SDValue N, SDValue &Imm) const;
107 virtual bool selectVSplatUimm4(SDValue N, SDValue &Imm) const;
109 virtual bool selectVSplatUimm5(SDValue N, SDValue &Imm) const;
109 virtual bool selectVSplatUimm5(SDValue N, SDValue &Imm) const;
111 virtual bool selectVSplatUimm6(SDValue N, SDValue &Imm) const;
111 virtual bool selectVSplatUimm6(SDValue N, SDValue &Imm) const;
113 virtual bool selectVSplatUimm8(SDValue N, SDValue &Imm) const;
113 virtual bool selectVSplatUimm8(SDValue N, SDValue &Imm) const;
115 virtual bool selectVSplatSimm5(SDValue N, SDValue &Imm) const;
115 virtual bool selectVSplatSimm5(SDValue N, SDValue &Imm) const;
117 virtual bool selectVSplatUimmPow2(SDValue N, SDValue &Imm) const;
117 virtual bool selectVSplatUimmPow2(SDValue N, SDValue &Imm) const;
120 virtual bool selectVSplatUimmInvPow2(SDValue N, SDValue &Imm) const;
120 virtual bool selectVSplatUimmInvPow2(SDValue N, SDValue &Imm) const;
123 virtual bool selectVSplatMaskL(SDValue N, SDValue &Imm) const;
123 virtual bool selectVSplatMaskL(SDValue N, SDValue &Imm) const;
126 virtual bool selectVSplatMaskR(SDValue N, SDValue &Imm) const;
126 virtual bool selectVSplatMaskR(SDValue N, SDValue &Imm) const;
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) {
144 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
146 std::vector<SDValue> &OutOps) override;
lib/Target/Mips/MipsISelLowering.cpp 146 SDValue MipsTargetLowering::getGlobalReg(SelectionDAG &DAG, EVT Ty) const {
151 SDValue MipsTargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
157 SDValue MipsTargetLowering::getTargetNode(ExternalSymbolSDNode *N, EVT Ty,
163 SDValue MipsTargetLowering::getTargetNode(BlockAddressSDNode *N, EVT Ty,
169 SDValue MipsTargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
175 SDValue MipsTargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
567 static SDValue performDivRemCombine(SDNode *N, SelectionDAG &DAG,
580 SDValue DivRem = DAG.getNode(Opc, DL, MVT::Glue,
582 SDValue InChain = DAG.getEntryNode();
583 SDValue InGlue = DivRem;
587 SDValue CopyFromLo = DAG.getCopyFromReg(InChain, DL, LO, Ty,
596 SDValue CopyFromHi = DAG.getCopyFromReg(InChain, DL,
644 static SDValue createFPCmp(SelectionDAG &DAG, const SDValue &Op) {
644 static SDValue createFPCmp(SelectionDAG &DAG, const SDValue &Op) {
649 SDValue LHS = Op.getOperand(0);
654 SDValue RHS = Op.getOperand(1);
666 static SDValue createCMovFP(SelectionDAG &DAG, SDValue Cond, SDValue True,
666 static SDValue createCMovFP(SelectionDAG &DAG, SDValue Cond, SDValue True,
666 static SDValue createCMovFP(SelectionDAG &DAG, SDValue Cond, SDValue True,
667 SDValue False, const SDLoc &DL) {
670 SDValue FCC0 = DAG.getRegister(Mips::FCC0, MVT::i32);
676 static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
682 SDValue SetCC = N->getOperand(0);
688 SDValue False = N->getOperand(2);
710 SDValue True = N->getOperand(1);
720 SDValue True = N->getOperand(1);
755 static SDValue performCMovFPCombine(SDNode *N, SelectionDAG &DAG,
761 SDValue ValueIfTrue = N->getOperand(0), ValueIfFalse = N->getOperand(2);
777 SDValue FCC = N->getOperand(1), Glue = N->getOperand(3);
782 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
788 SDValue FirstOperand = N->getOperand(0);
790 SDValue Mask = N->getOperand(1);
796 SDValue NewOperand;
864 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
874 SDValue And0 = N->getOperand(0), And1 = N->getOperand(1);
898 SDValue Shl = And1.getOperand(0);
940 SDValue Const1;
941 SDValue SrlX;
962 static SDValue performMADD_MSUBCombine(SDNode *ROOTNode, SelectionDAG &CurDAG,
996 SDValue Mult = ROOTNode->getOperand(0).getOpcode() == ISD::MUL
1000 SDValue AddOperand = ROOTNode->getOperand(0).getOpcode() == ISD::MUL
1014 SDValue MultLHS = Mult->getOperand(0);
1015 SDValue MultRHS = Mult->getOperand(1);
1027 SDValue TopHalf;
1028 SDValue BottomHalf;
1034 SDValue ACCIn = CurDAG.getNode(MipsISD::MTLOHI, DL, MVT::Untyped,
1042 SDValue MAddOps[3] = {
1046 SDValue MAdd = CurDAG.getNode(Opcode, DL, VTs, MAddOps);
1048 SDValue ResLo = CurDAG.getNode(MipsISD::MFLO, DL, MVT::i32, MAdd);
1049 SDValue ResHi = CurDAG.getNode(MipsISD::MFHI, DL, MVT::i32, MAdd);
1050 SDValue Combined =
1055 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
1070 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
1083 SDValue Add = N->getOperand(1);
1088 SDValue Lo = Add.getOperand(1);
1097 SDValue Add1 = DAG.getNode(ISD::ADD, DL, ValTy, N->getOperand(0),
1102 static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
1112 SDValue FirstOperand = N->getOperand(0);
1114 SDValue SecondOperand = N->getOperand(1);
1120 SDValue NewOperand;
1153 SDValue MipsTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
1200 SmallVectorImpl<SDValue> &Results,
1202 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
1211 SmallVectorImpl<SDValue> &Results,
1216 SDValue MipsTargetLowering::
1217 LowerOperation(SDValue Op, SelectionDAG &DAG) const
1901 SDValue MipsTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1901 SDValue MipsTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1904 SDValue Chain = Op.getOperand(0);
1905 SDValue Dest = Op.getOperand(2);
1909 SDValue CondRes = createFPCmp(DAG, Op.getOperand(1));
1915 SDValue CCNode = CondRes.getOperand(2);
1919 SDValue BrCode = DAG.getConstant(Opc, DL, MVT::i32);
1920 SDValue FCC0 = DAG.getRegister(Mips::FCC0, MVT::i32);
1925 SDValue MipsTargetLowering::
1926 lowerSELECT(SDValue Op, SelectionDAG &DAG) const
1929 SDValue Cond = createFPCmp(DAG, Op.getOperand(0));
1939 SDValue MipsTargetLowering::lowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1939 SDValue MipsTargetLowering::lowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1941 SDValue Cond = createFPCmp(DAG, Op);
1947 SDValue True = DAG.getConstant(1, DL, MVT::i32);
1948 SDValue False = DAG.getConstant(0, DL, MVT::i32);
1953 SDValue MipsTargetLowering::lowerGlobalAddress(SDValue Op,
1953 SDValue MipsTargetLowering::lowerGlobalAddress(SDValue Op,
2000 SDValue MipsTargetLowering::lowerBlockAddress(SDValue Op,
2000 SDValue MipsTargetLowering::lowerBlockAddress(SDValue Op,
2012 SDValue MipsTargetLowering::
2013 lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
2034 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, Flag);
2035 SDValue Argument = DAG.getNode(MipsISD::Wrapper, DL, PtrVT,
2040 SDValue TlsGetAddr = DAG.getExternalSymbol("__tls_get_addr", PtrVT);
2052 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2052 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2054 SDValue Ret = CallResult.first;
2059 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2061 SDValue Hi = DAG.getNode(MipsISD::TlsHi, DL, PtrVT, TGAHi);
2062 SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2064 SDValue Lo = DAG.getNode(MipsISD::Lo, DL, PtrVT, TGALo);
2065 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Ret);
2069 SDValue Offset;
2072 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2081 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2083 SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2085 SDValue Hi = DAG.getNode(MipsISD::TlsHi, DL, PtrVT, TGAHi);
2086 SDValue Lo = DAG.getNode(MipsISD::Lo, DL, PtrVT, TGALo);
2090 SDValue ThreadPointer = DAG.getNode(MipsISD::ThreadPointer, DL, PtrVT);
2094 SDValue MipsTargetLowering::
2095 lowerJumpTable(SDValue Op, SelectionDAG &DAG) const
2107 SDValue MipsTargetLowering::
2108 lowerConstantPool(SDValue Op, SelectionDAG &DAG) const
2130 SDValue MipsTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
2130 SDValue MipsTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
2135 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
2145 SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
2145 SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
2148 SDValue Chain = Node->getOperand(0);
2149 SDValue VAListPtr = Node->getOperand(1);
2156 SDValue VAListLoad = DAG.getLoad(getPointerTy(DAG.getDataLayout()), DL, Chain,
2158 SDValue VAList = VAListLoad;
2181 SDValue Tmp3 =
2203 static SDValue lowerFCOPYSIGN32(SDValue Op, SelectionDAG &DAG,
2203 static SDValue lowerFCOPYSIGN32(SDValue Op, SelectionDAG &DAG,
2208 SDValue Const1 = DAG.getConstant(1, DL, MVT::i32);
2209 SDValue Const31 = DAG.getConstant(31, DL, MVT::i32);
2210 SDValue Res;
2214 SDValue X = (TyX == MVT::f32) ?
2218 SDValue Y = (TyY == MVT::f32) ?
2226 SDValue E = DAG.getNode(MipsISD::Ext, DL, MVT::i32, Y, Const31, Const1);
2234 SDValue SllX = DAG.getNode(ISD::SHL, DL, MVT::i32, X, Const1);
2235 SDValue SrlX = DAG.getNode(ISD::SRL, DL, MVT::i32, SllX, Const1);
2236 SDValue SrlY = DAG.getNode(ISD::SRL, DL, MVT::i32, Y, Const31);
2237 SDValue SllY = DAG.getNode(ISD::SHL, DL, MVT::i32, SrlY, Const31);
2244 SDValue LowX = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
2250 static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
2250 static SDValue lowerFCOPYSIGN64(SDValue Op, SelectionDAG &DAG,
2256 SDValue Const1 = DAG.getConstant(1, DL, MVT::i32);
2259 SDValue X = DAG.getNode(ISD::BITCAST, DL, TyX, Op.getOperand(0));
2260 SDValue Y = DAG.getNode(ISD::BITCAST, DL, TyY, Op.getOperand(1));
2265 SDValue E = DAG.getNode(MipsISD::Ext, DL, TyY, Y,
2273 SDValue I = DAG.getNode(MipsISD::Ins, DL, TyX, E,
2284 SDValue SllX = DAG.getNode(ISD::SHL, DL, TyX, X, Const1);
2285 SDValue SrlX = DAG.getNode(ISD::SRL, DL, TyX, SllX, Const1);
2286 SDValue SrlY = DAG.getNode(ISD::SRL, DL, TyY, Y,
2294 SDValue SllY = DAG.getNode(ISD::SHL, DL, TyX, SrlY,
2296 SDValue Or = DAG.getNode(ISD::OR, DL, TyX, SrlX, SllY);
2300 SDValue
2301 MipsTargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
2308 static SDValue lowerFABS32(SDValue Op, SelectionDAG &DAG,
2308 static SDValue lowerFABS32(SDValue Op, SelectionDAG &DAG,
2311 SDValue Res, Const1 = DAG.getConstant(1, DL, MVT::i32);
2315 SDValue X = (Op.getValueType() == MVT::f32)
2328 SDValue SllX = DAG.getNode(ISD::SHL, DL, MVT::i32, X, Const1);
2339 SDValue LowX =
2345 static SDValue lowerFABS64(SDValue Op, SelectionDAG &DAG,
2345 static SDValue lowerFABS64(SDValue Op, SelectionDAG &DAG,
2348 SDValue Res, Const1 = DAG.getConstant(1, DL, MVT::i32);
2351 SDValue X = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Op.getOperand(0));
2359 SDValue SllX = DAG.getNode(ISD::SHL, DL, MVT::i64, X, Const1);
2366 SDValue MipsTargetLowering::lowerFABS(SDValue Op, SelectionDAG &DAG) const {
2366 SDValue MipsTargetLowering::lowerFABS(SDValue Op, SelectionDAG &DAG) const {
2373 SDValue MipsTargetLowering::
2374 lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
2386 SDValue FrameAddr = DAG.getCopyFromReg(
2391 SDValue MipsTargetLowering::lowerRETURNADDR(SDValue Op,
2391 SDValue MipsTargetLowering::lowerRETURNADDR(SDValue Op,
2418 SDValue MipsTargetLowering::lowerEH_RETURN(SDValue Op, SelectionDAG &DAG)
2418 SDValue MipsTargetLowering::lowerEH_RETURN(SDValue Op, SelectionDAG &DAG)
2424 SDValue Chain = Op.getOperand(0);
2425 SDValue Offset = Op.getOperand(1);
2426 SDValue Handler = Op.getOperand(2);
2442 SDValue MipsTargetLowering::lowerATOMIC_FENCE(SDValue Op,
2442 SDValue MipsTargetLowering::lowerATOMIC_FENCE(SDValue Op,
2452 SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op,
2452 SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op,
2457 SDValue Lo = Op.getOperand(0), Hi = Op.getOperand(1);
2458 SDValue Shamt = Op.getOperand(2);
2465 SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
2467 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo,
2469 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, Not);
2470 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
2471 SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
2472 SDValue ShiftLeftLo = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
2473 SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
2479 SDValue Ops[2] = {Lo, Hi};
2483 SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
2483 SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
2486 SDValue Lo = Op.getOperand(0), Hi = Op.getOperand(1);
2487 SDValue Shamt = Op.getOperand(2);
2503 SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt,
2505 SDValue ShiftLeft1Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
2507 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, ShiftLeft1Hi, Not);
2508 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
2509 SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
2510 SDValue ShiftRightHi = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL,
2512 SDValue Cond = DAG.getNode(ISD::AND, DL, MVT::i32, Shamt,
2514 SDValue Ext = DAG.getNode(ISD::SRA, DL, VT, Hi,
2530 SDValue Ops[2] = {Lo, Hi};
2534 static SDValue createLoadLR(unsigned Opc, SelectionDAG &DAG, LoadSDNode *LD,
2535 SDValue Chain, SDValue Src, unsigned Offset) {
2535 SDValue Chain, SDValue Src, unsigned Offset) {
2536 SDValue Ptr = LD->getBasePtr();
2546 SDValue Ops[] = { Chain, Ptr, Src };
2552 SDValue MipsTargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
2552 SDValue MipsTargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
2567 SDValue Chain = LD->getChain(), Undef = DAG.getUNDEF(VT);
2577 SDValue LDL = createLoadLR(MipsISD::LDL, DAG, LD, Chain, Undef,
2583 SDValue LWL = createLoadLR(MipsISD::LWL, DAG, LD, Chain, Undef,
2585 SDValue LWR = createLoadLR(MipsISD::LWR, DAG, LD, LWL.getValue(1), LWL,
2609 SDValue Const32 = DAG.getConstant(32, DL, MVT::i32);
2610 SDValue SLL = DAG.getNode(ISD::SHL, DL, MVT::i64, LWR, Const32);
2611 SDValue SRL = DAG.getNode(ISD::SRL, DL, MVT::i64, SLL, Const32);
2612 SDValue Ops[] = { SRL, LWR.getValue(1) };
2616 static SDValue createStoreLR(unsigned Opc, SelectionDAG &DAG, StoreSDNode *SD,
2617 SDValue Chain, unsigned Offset) {
2618 SDValue Ptr = SD->getBasePtr(), Value = SD->getValue();
2627 SDValue Ops[] = { Chain, Value, Ptr };
2633 static SDValue lowerUnalignedIntStore(StoreSDNode *SD, SelectionDAG &DAG,
2635 SDValue Value = SD->getValue(), Chain = SD->getChain();
2645 SDValue SWL = createStoreLR(MipsISD::SWL, DAG, SD, Chain,
2657 SDValue SDL = createStoreLR(MipsISD::SDL, DAG, SD, Chain, IsLittle ? 7 : 0);
2662 static SDValue lowerFP_TO_SINT_STORE(StoreSDNode *SD, SelectionDAG &DAG,
2664 SDValue Val = SD->getValue();
2671 SDValue Tr = DAG.getNode(MipsISD::TruncIntFP, SDLoc(Val), FPTy,
2678 SDValue MipsTargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
2678 SDValue MipsTargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
2691 SDValue MipsTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
2691 SDValue MipsTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
2702 SDValue MipsTargetLowering::lowerFP_TO_SINT(SDValue Op,
2702 SDValue MipsTargetLowering::lowerFP_TO_SINT(SDValue Op,
2708 SDValue Trunc = DAG.getNode(MipsISD::TruncIntFP, SDLoc(Op), FPTy,
2887 SDValue MipsTargetLowering::passArgOnStack(SDValue StackPtr, unsigned Offset,
2887 SDValue MipsTargetLowering::passArgOnStack(SDValue StackPtr, unsigned Offset,
2888 SDValue Chain, SDValue Arg,
2888 SDValue Chain, SDValue Arg,
2892 SDValue PtrOff =
2900 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2906 getOpndList(SmallVectorImpl<SDValue> &Ops,
2907 std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
2909 bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
2910 SDValue Chain) const {
2931 SDValue InFlag;
2992 const SDValue TargetAddr = Node->getOperand(0).getOperand(1);
3015 SDValue
3017 SmallVectorImpl<SDValue> &InVals) const {
3021 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
3023 SDValue Chain = CLI.Chain;
3024 SDValue Callee = CLI.Callee;
3111 SDValue NextStackOffsetVal = DAG.getIntPtrConstant(NextStackOffset, DL, true);
3116 SDValue StackPtr =
3120 std::deque<std::pair<unsigned, SDValue>> RegsToPass;
3121 SmallVector<SDValue, 8> MemOpChains;
3127 SDValue Arg = OutVals[i];
3162 SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
3164 SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
3307 SmallVector<SDValue, 8> Ops(1, Chain);
3319 SDValue InFlag = Chain.getValue(1);
3337 SDValue MipsTargetLowering::LowerCallResult(
3338 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
3338 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
3340 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
3357 SDValue Val = DAG.getCopyFromReg(Chain, DL, RVLocs[i].getLocReg(),
3404 static SDValue UnpackFromArgumentSlot(SDValue Val, const CCValAssign &VA,
3404 static SDValue UnpackFromArgumentSlot(SDValue Val, const CCValAssign &VA,
3464 SDValue MipsTargetLowering::LowerFormalArguments(
3465 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
3467 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3475 std::vector<SDValue> OutChains;
3530 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
3544 SDValue ArgValue2 = DAG.getCopyFromReg(Chain, DL, Reg2, RegVT);
3572 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3573 SDValue ArgValue = DAG.getLoad(
3595 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[i]);
3636 SDValue
3637 MipsTargetLowering::LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
3648 SDValue
3649 MipsTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3652 const SmallVectorImpl<SDValue> &OutVals,
3665 SDValue Flag;
3666 SmallVector<SDValue, 4> RetOps(1, Chain);
3670 SDValue Val = OutVals[i];
3728 SDValue Val =
4020 void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4022 std::vector<SDValue>&Ops,
4025 SDValue Result;
4174 SDValue Chain, const SDLoc &DL, std::vector<SDValue> &OutChains,
4174 SDValue Chain, const SDLoc &DL, std::vector<SDValue> &OutChains,
4176 SmallVectorImpl<SDValue> &InVals, const Argument *FuncArg,
4203 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
4217 SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrTy, FIN,
4219 SDValue Store = DAG.getStore(Chain, DL, DAG.getRegister(VReg, RegTy),
4227 SDValue Chain, const SDLoc &DL,
4228 std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
4229 SmallVectorImpl<SDValue> &MemOpChains, SDValue StackPtr,
4229 SmallVectorImpl<SDValue> &MemOpChains, SDValue StackPtr,
4230 MachineFrameInfo &MFI, SelectionDAG &DAG, SDValue Arg, unsigned FirstReg,
4248 SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
4250 SDValue LoadVal = DAG.getLoad(RegTy, DL, Chain, LoadPtr,
4263 SDValue Val;
4273 SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
4276 SDValue LoadVal = DAG.getExtLoad(
4289 SDValue Shift = DAG.getNode(ISD::SHL, DL, RegTy, LoadVal,
4310 SDValue Src = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
4312 SDValue Dst = DAG.getNode(ISD::ADD, DL, PtrTy, StackPtr,
4322 void MipsTargetLowering::writeVarArgRegs(std::vector<SDValue> &OutChains,
4323 SDValue Chain, const SDLoc &DL,
4358 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegTy);
4360 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4361 SDValue Store =
lib/Target/Mips/MipsISelLowering.h 320 SmallVectorImpl<SDValue> &Results,
324 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
324 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
329 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
340 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
385 SDValue getGlobalReg(SelectionDAG &DAG, EVT Ty) const;
392 SDValue getAddrLocal(NodeTy *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG,
395 SDValue GOT = DAG.getNode(MipsISD::Wrapper, DL, Ty, getGlobalReg(DAG, Ty),
397 SDValue Load =
401 SDValue Lo = DAG.getNode(MipsISD::Lo, DL, Ty,
411 SDValue getAddrGlobal(NodeTy *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG,
412 unsigned Flag, SDValue Chain,
414 SDValue Tgt = DAG.getNode(MipsISD::Wrapper, DL, Ty, getGlobalReg(DAG, Ty),
424 SDValue getAddrGlobalLargeGOT(NodeTy *N, const SDLoc &DL, EVT Ty,
426 unsigned LoFlag, SDValue Chain,
428 SDValue Hi = DAG.getNode(MipsISD::GotHi, DL, Ty,
431 SDValue Wrapper = DAG.getNode(MipsISD::Wrapper, DL, Ty, Hi,
443 SDValue getAddrNonPIC(NodeTy *N, const SDLoc &DL, EVT Ty,
445 SDValue Hi = getTargetNode(N, Ty, DAG, MipsII::MO_ABS_HI);
446 SDValue Lo = getTargetNode(N, Ty, DAG, MipsII::MO_ABS_LO);
460 SDValue getAddrNonPICSym64(NodeTy *N, const SDLoc &DL, EVT Ty,
462 SDValue Hi = getTargetNode(N, Ty, DAG, MipsII::MO_ABS_HI);
463 SDValue Lo = getTargetNode(N, Ty, DAG, MipsII::MO_ABS_LO);
465 SDValue Highest =
468 SDValue Higher = getTargetNode(N, Ty, DAG, MipsII::MO_HIGHER);
469 SDValue HigherPart =
472 SDValue Cst = DAG.getConstant(16, DL, MVT::i32);
473 SDValue Shift = DAG.getNode(ISD::SHL, DL, Ty, HigherPart, Cst);
474 SDValue Add = DAG.getNode(ISD::ADD, DL, Ty, Shift,
476 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, Ty, Add, Cst);
487 SDValue getAddrGPRel(NodeTy *N, const SDLoc &DL, EVT Ty,
489 SDValue GPRel = getTargetNode(N, Ty, DAG, MipsII::MO_GPREL);
500 getOpndList(SmallVectorImpl<SDValue> &Ops,
501 std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
503 bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
504 SDValue Chain) const;
507 SDValue lowerLOAD(SDValue Op, SelectionDAG &DAG) const;
507 SDValue lowerLOAD(SDValue Op, SelectionDAG &DAG) const;
508 SDValue lowerSTORE(SDValue Op, SelectionDAG &DAG) const;
508 SDValue lowerSTORE(SDValue Op, SelectionDAG &DAG) const;
517 SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
521 SDValue getTargetNode(ExternalSymbolSDNode *N, EVT Ty, SelectionDAG &DAG,
525 SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
529 SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG,
533 SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG,
537 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
537 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
537 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
541 SmallVectorImpl<SDValue> &InVals,
545 SDValue lowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
545 SDValue lowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
546 SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
546 SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
547 SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
547 SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
548 SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
548 SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
549 SDValue lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
549 SDValue lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
550 SDValue lowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
550 SDValue lowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
551 SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
551 SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
552 SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
552 SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
553 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
553 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
554 SDValue lowerVAARG(SDValue Op, SelectionDAG &DAG) const;
554 SDValue lowerVAARG(SDValue Op, SelectionDAG &DAG) const;
555 SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
555 SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
556 SDValue lowerFABS(SDValue Op, SelectionDAG &DAG) const;
556 SDValue lowerFABS(SDValue Op, SelectionDAG &DAG) const;
557 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
557 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
558 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
558 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
559 SDValue lowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
559 SDValue lowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
560 SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const;
560 SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const;
561 SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG& DAG) const;
561 SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG& DAG) const;
562 SDValue lowerShiftRightParts(SDValue Op, SelectionDAG& DAG,
562 SDValue lowerShiftRightParts(SDValue Op, SelectionDAG& DAG,
564 SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
564 SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
565 SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
565 SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
577 void copyByValRegs(SDValue Chain, const SDLoc &DL,
578 std::vector<SDValue> &OutChains, SelectionDAG &DAG,
580 SmallVectorImpl<SDValue> &InVals,
586 void passByValArg(SDValue Chain, const SDLoc &DL,
587 std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
588 SmallVectorImpl<SDValue> &MemOpChains, SDValue StackPtr,
588 SmallVectorImpl<SDValue> &MemOpChains, SDValue StackPtr,
589 MachineFrameInfo &MFI, SelectionDAG &DAG, SDValue Arg,
597 void writeVarArgRegs(std::vector<SDValue> &OutChains, SDValue Chain,
597 void writeVarArgRegs(std::vector<SDValue> &OutChains, SDValue Chain,
601 SDValue
602 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
605 SmallVectorImpl<SDValue> &InVals) const override;
607 SDValue passArgOnStack(SDValue StackPtr, unsigned Offset, SDValue Chain,
607 SDValue passArgOnStack(SDValue StackPtr, unsigned Offset, SDValue Chain,
607 SDValue passArgOnStack(SDValue StackPtr, unsigned Offset, SDValue Chain,
608 SDValue Arg, const SDLoc &DL, bool IsTailCall,
611 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
612 SmallVectorImpl<SDValue> &InVals) const override;
619 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
619 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
621 const SmallVectorImpl<SDValue> &OutVals,
624 SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
624 SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
650 void LowerAsmOperandForConstraint(SDValue Op,
652 std::vector<SDValue> &Ops,
lib/Target/Mips/MipsSEISelDAGToDAG.cpp 77 unsigned MipsSEDAGToDAGISel::getMSACtrlReg(const SDValue RegIdx) const {
206 SDValue InFlag = Node->getOperand(2);
208 SDValue LHS = Node->getOperand(0), RHS = Node->getOperand(1);
214 SDValue Ops[3] = {LHS, RHS, InFlag};
233 SDValue CstOne = CurDAG->getTargetConstant(1, DL, MVT::i32);
235 SDValue OuFlag = CurDAG->getTargetConstant(20, DL, MVT::i32);
243 SDValue Ops[4] = {SDValue(DSPCtrlField, 0),
253 SDValue Zero = CurDAG->getRegister(Mips::ZERO, MVT::i32);
255 SDValue InsOps[4] = {Zero, OuFlag, CstOne, SDValue(DSPCFWithCarry, 0)};
261 SDValue Operands[3] = {LHS, RHS, SDValue(WrDSP, 0)};
266 bool MipsSEDAGToDAGISel::selectAddrFrameIndex(SDValue Addr, SDValue &Base,
266 bool MipsSEDAGToDAGISel::selectAddrFrameIndex(SDValue Addr, SDValue &Base,
267 SDValue &Offset) const {
280 SDValue Addr, SDValue &Base, SDValue &Offset, unsigned OffsetBits,
280 SDValue Addr, SDValue &Base, SDValue &Offset, unsigned OffsetBits,
280 SDValue Addr, SDValue &Base, SDValue &Offset, unsigned OffsetBits,
310 bool MipsSEDAGToDAGISel::selectAddrRegImm(SDValue Addr, SDValue &Base,
310 bool MipsSEDAGToDAGISel::selectAddrRegImm(SDValue Addr, SDValue &Base,
311 SDValue &Offset) const {
345 SDValue Opnd0 = Addr.getOperand(1).getOperand(0);
360 bool MipsSEDAGToDAGISel::selectAddrDefault(SDValue Addr, SDValue &Base,
360 bool MipsSEDAGToDAGISel::selectAddrDefault(SDValue Addr, SDValue &Base,
361 SDValue &Offset) const {
367 bool MipsSEDAGToDAGISel::selectIntAddr(SDValue Addr, SDValue &Base,
367 bool MipsSEDAGToDAGISel::selectIntAddr(SDValue Addr, SDValue &Base,
368 SDValue &Offset) const {
373 bool MipsSEDAGToDAGISel::selectAddrRegImm9(SDValue Addr, SDValue &Base,
373 bool MipsSEDAGToDAGISel::selectAddrRegImm9(SDValue Addr, SDValue &Base,
374 SDValue &Offset) const {
385 bool MipsSEDAGToDAGISel::selectAddrRegImm11(SDValue Addr, SDValue &Base,
385 bool MipsSEDAGToDAGISel::selectAddrRegImm11(SDValue Addr, SDValue &Base,
386 SDValue &Offset) const {
397 bool MipsSEDAGToDAGISel::selectAddrRegImm12(SDValue Addr, SDValue &Base,
397 bool MipsSEDAGToDAGISel::selectAddrRegImm12(SDValue Addr, SDValue &Base,
398 SDValue &Offset) const {
408 bool MipsSEDAGToDAGISel::selectAddrRegImm16(SDValue Addr, SDValue &Base,
408 bool MipsSEDAGToDAGISel::selectAddrRegImm16(SDValue Addr, SDValue &Base,
409 SDValue &Offset) const {
419 bool MipsSEDAGToDAGISel::selectIntAddr11MM(SDValue Addr, SDValue &Base,
419 bool MipsSEDAGToDAGISel::selectIntAddr11MM(SDValue Addr, SDValue &Base,
420 SDValue &Offset) const {
425 bool MipsSEDAGToDAGISel::selectIntAddr12MM(SDValue Addr, SDValue &Base,
425 bool MipsSEDAGToDAGISel::selectIntAddr12MM(SDValue Addr, SDValue &Base,
426 SDValue &Offset) const {
431 bool MipsSEDAGToDAGISel::selectIntAddr16MM(SDValue Addr, SDValue &Base,
431 bool MipsSEDAGToDAGISel::selectIntAddr16MM(SDValue Addr, SDValue &Base,
432 SDValue &Offset) const {
437 bool MipsSEDAGToDAGISel::selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
437 bool MipsSEDAGToDAGISel::selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
438 SDValue &Offset) const {
459 bool MipsSEDAGToDAGISel::selectIntAddrSImm10(SDValue Addr, SDValue &Base,
459 bool MipsSEDAGToDAGISel::selectIntAddrSImm10(SDValue Addr, SDValue &Base,
460 SDValue &Offset) const {
471 bool MipsSEDAGToDAGISel::selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
471 bool MipsSEDAGToDAGISel::selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
472 SDValue &Offset) const {
482 bool MipsSEDAGToDAGISel::selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
482 bool MipsSEDAGToDAGISel::selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
483 SDValue &Offset) const {
493 bool MipsSEDAGToDAGISel::selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
493 bool MipsSEDAGToDAGISel::selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
494 SDValue &Offset) const {
549 selectVSplatCommon(SDValue N, SDValue &Imm, bool Signed,
549 selectVSplatCommon(SDValue N, SDValue &Imm, bool Signed,
572 selectVSplatUimm1(SDValue N, SDValue &Imm) const {
572 selectVSplatUimm1(SDValue N, SDValue &Imm) const {
577 selectVSplatUimm2(SDValue N, SDValue &Imm) const {
577 selectVSplatUimm2(SDValue N, SDValue &Imm) const {
582 selectVSplatUimm3(SDValue N, SDValue &Imm) const {
582 selectVSplatUimm3(SDValue N, SDValue &Imm) const {
588 selectVSplatUimm4(SDValue N, SDValue &Imm) const {
588 selectVSplatUimm4(SDValue N, SDValue &Imm) const {
594 selectVSplatUimm5(SDValue N, SDValue &Imm) const {
594 selectVSplatUimm5(SDValue N, SDValue &Imm) const {
600 selectVSplatUimm6(SDValue N, SDValue &Imm) const {
600 selectVSplatUimm6(SDValue N, SDValue &Imm) const {
606 selectVSplatUimm8(SDValue N, SDValue &Imm) const {
606 selectVSplatUimm8(SDValue N, SDValue &Imm) const {
612 selectVSplatSimm5(SDValue N, SDValue &Imm) const {
612 selectVSplatSimm5(SDValue N, SDValue &Imm) const {
626 bool MipsSEDAGToDAGISel::selectVSplatUimmPow2(SDValue N, SDValue &Imm) const {
626 bool MipsSEDAGToDAGISel::selectVSplatUimmPow2(SDValue N, SDValue &Imm) const {
657 bool MipsSEDAGToDAGISel::selectVSplatMaskL(SDValue N, SDValue &Imm) const {
657 bool MipsSEDAGToDAGISel::selectVSplatMaskL(SDValue N, SDValue &Imm) const {
691 bool MipsSEDAGToDAGISel::selectVSplatMaskR(SDValue N, SDValue &Imm) const {
691 bool MipsSEDAGToDAGISel::selectVSplatMaskR(SDValue N, SDValue &Imm) const {
712 bool MipsSEDAGToDAGISel::selectVSplatUimmInvPow2(SDValue N,
713 SDValue &Imm) const {
747 SDValue cond = Node->getOperand(0);
748 SDValue Hi1 = Node->getOperand(1);
749 SDValue Lo1 = Node->getOperand(2);
750 SDValue Hi2 = Node->getOperand(3);
751 SDValue Lo2 = Node->getOperand(4);
753 SDValue ops[] = {cond, Hi1, Lo1, Hi2, Lo2};
771 SDValue Zero = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
776 SDValue Zero = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
781 SDValue Zero = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
807 SDValue ImmOpnd = CurDAG->getTargetConstant(SignExtend64<16>(Inst->ImmOpnd),
839 SDValue ChainIn = Node->getOperand(0);
840 SDValue RegIdx = Node->getOperand(2);
841 SDValue Reg = CurDAG->getCopyFromReg(ChainIn, DL,
872 SDValue ChainIn = Node->getOperand(0);
873 SDValue RegIdx = Node->getOperand(2);
874 SDValue Value = Node->getOperand(3);
875 SDValue ChainOut = CurDAG->getCopyToReg(ChainIn, DL,
931 SDValue Ops[4] = {
958 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, DestReg,
960 SDValue ResNode = CurDAG->getCopyFromReg(Chain, DL, DestReg, PtrVT);
1025 SDValue Imm = CurDAG->getTargetConstant(SplatValue, DL,
1040 SDValue ZeroVal = CurDAG->getRegister(
1054 SDValue LoVal = CurDAG->getTargetConstant(Lo, DL, SplatMVT);
1064 SDValue ZeroVal = CurDAG->getRegister(Mips::ZERO, MVT::i32);
1066 SDValue LoVal = CurDAG->getTargetConstant(Lo, DL, MVT::i32);
1067 SDValue HiVal = CurDAG->getTargetConstant(Hi, DL, MVT::i32);
1086 SDValue ZeroVal = CurDAG->getRegister(Mips::ZERO, MVT::i32);
1088 SDValue LoVal = CurDAG->getTargetConstant(Lo, DL, MVT::i32);
1089 SDValue HiVal = CurDAG->getTargetConstant(Hi, DL, MVT::i32);
1138 SDValue LoVal = CurDAG->getTargetConstant(Lo, DL, MVT::i32);
1139 SDValue HiVal = CurDAG->getTargetConstant(Hi, DL, MVT::i32);
1140 SDValue HigherVal = CurDAG->getTargetConstant(Higher, DL, MVT::i32);
1141 SDValue HighestVal = CurDAG->getTargetConstant(Highest, DL, MVT::i32);
1142 SDValue ZeroVal = CurDAG->getRegister(Mips::ZERO, MVT::i32);
1199 SDValue Zero64Val = CurDAG->getRegister(Mips::ZERO_64, MVT::i64);
1228 SDValue Ops[4] = {HiResNonZero ? SDValue(HiRes, 0) : Zero64Val,
1273 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
1274 std::vector<SDValue> &OutOps) {
1275 SDValue Base, Offset;
lib/Target/Mips/MipsSEISelDAGToDAG.h 35 unsigned getMSACtrlReg(const SDValue RegIdx) const;
45 bool selectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset) const;
45 bool selectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset) const;
45 bool selectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset) const;
46 bool selectAddrFrameIndexOffset(SDValue Addr, SDValue &Base, SDValue &Offset,
46 bool selectAddrFrameIndexOffset(SDValue Addr, SDValue &Base, SDValue &Offset,
46 bool selectAddrFrameIndexOffset(SDValue Addr, SDValue &Base, SDValue &Offset,
50 bool selectAddrRegImm(SDValue Addr, SDValue &Base,
50 bool selectAddrRegImm(SDValue Addr, SDValue &Base,
51 SDValue &Offset) const override;
53 bool selectAddrDefault(SDValue Addr, SDValue &Base,
53 bool selectAddrDefault(SDValue Addr, SDValue &Base,
54 SDValue &Offset) const override;
56 bool selectIntAddr(SDValue Addr, SDValue &Base,
56 bool selectIntAddr(SDValue Addr, SDValue &Base,
57 SDValue &Offset) const override;
59 bool selectAddrRegImm9(SDValue Addr, SDValue &Base,
59 bool selectAddrRegImm9(SDValue Addr, SDValue &Base,
60 SDValue &Offset) const;
62 bool selectAddrRegImm11(SDValue Addr, SDValue &Base,
62 bool selectAddrRegImm11(SDValue Addr, SDValue &Base,
63 SDValue &Offset) const;
65 bool selectAddrRegImm12(SDValue Addr, SDValue &Base,
65 bool selectAddrRegImm12(SDValue Addr, SDValue &Base,
66 SDValue &Offset) const;
68 bool selectAddrRegImm16(SDValue Addr, SDValue &Base,
68 bool selectAddrRegImm16(SDValue Addr, SDValue &Base,
69 SDValue &Offset) const;
71 bool selectIntAddr11MM(SDValue Addr, SDValue &Base,
71 bool selectIntAddr11MM(SDValue Addr, SDValue &Base,
72 SDValue &Offset) const override;
74 bool selectIntAddr12MM(SDValue Addr, SDValue &Base,
74 bool selectIntAddr12MM(SDValue Addr, SDValue &Base,
75 SDValue &Offset) const override;
77 bool selectIntAddr16MM(SDValue Addr, SDValue &Base,
77 bool selectIntAddr16MM(SDValue Addr, SDValue &Base,
78 SDValue &Offset) const override;
80 bool selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
80 bool selectIntAddrLSL2MM(SDValue Addr, SDValue &Base,
81 SDValue &Offset) const override;
83 bool selectIntAddrSImm10(SDValue Addr, SDValue &Base,
83 bool selectIntAddrSImm10(SDValue Addr, SDValue &Base,
84 SDValue &Offset) const override;
86 bool selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
86 bool selectIntAddrSImm10Lsl1(SDValue Addr, SDValue &Base,
87 SDValue &Offset) const override;
89 bool selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
89 bool selectIntAddrSImm10Lsl2(SDValue Addr, SDValue &Base,
90 SDValue &Offset) const override;
92 bool selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
92 bool selectIntAddrSImm10Lsl3(SDValue Addr, SDValue &Base,
93 SDValue &Offset) const override;
99 bool selectVSplatCommon(SDValue N, SDValue &Imm, bool Signed,
99 bool selectVSplatCommon(SDValue N, SDValue &Imm, bool Signed,
102 bool selectVSplatUimm1(SDValue N, SDValue &Imm) const override;
102 bool selectVSplatUimm1(SDValue N, SDValue &Imm) const override;
104 bool selectVSplatUimm2(SDValue N, SDValue &Imm) const override;
104 bool selectVSplatUimm2(SDValue N, SDValue &Imm) const override;
106 bool selectVSplatUimm3(SDValue N, SDValue &Imm) const override;
106 bool selectVSplatUimm3(SDValue N, SDValue &Imm) const override;
108 bool selectVSplatUimm4(SDValue N, SDValue &Imm) const override;
108 bool selectVSplatUimm4(SDValue N, SDValue &Imm) const override;
110 bool selectVSplatUimm5(SDValue N, SDValue &Imm) const override;
110 bool selectVSplatUimm5(SDValue N, SDValue &Imm) const override;
112 bool selectVSplatUimm6(SDValue N, SDValue &Imm) const override;
112 bool selectVSplatUimm6(SDValue N, SDValue &Imm) const override;
114 bool selectVSplatUimm8(SDValue N, SDValue &Imm) const override;
114 bool selectVSplatUimm8(SDValue N, SDValue &Imm) const override;
116 bool selectVSplatSimm5(SDValue N, SDValue &Imm) const override;
116 bool selectVSplatSimm5(SDValue N, SDValue &Imm) const override;
118 bool selectVSplatUimmPow2(SDValue N, SDValue &Imm) const override;
118 bool selectVSplatUimmPow2(SDValue N, SDValue &Imm) const override;
121 bool selectVSplatUimmInvPow2(SDValue N, SDValue &Imm) const override;
121 bool selectVSplatUimmInvPow2(SDValue N, SDValue &Imm) const override;
124 bool selectVSplatMaskL(SDValue N, SDValue &Imm) const override;
124 bool selectVSplatMaskL(SDValue N, SDValue &Imm) const override;
127 bool selectVSplatMaskR(SDValue N, SDValue &Imm) const override;
127 bool selectVSplatMaskR(SDValue N, SDValue &Imm) const override;
137 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
139 std::vector<SDValue> &OutOps) override;
lib/Target/Mips/MipsSEISelLowering.cpp 408 SDValue MipsSETargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
408 SDValue MipsSETargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
418 SDValue Tmp = DAG.getNode(MipsISD::MTC1_D64, DL, MVT::f64, Op->getOperand(0));
448 SDValue MipsSETargetLowering::LowerOperation(SDValue Op,
448 SDValue MipsSETargetLowering::LowerOperation(SDValue Op,
481 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
487 SDValue Op0 = N->getOperand(0);
488 SDValue Op1 = N->getOperand(1);
507 SDValue Op0Op2 = Op0->getOperand(2);
514 SDValue Ops[] = { Op0->getOperand(0), Op0->getOperand(1), Op0Op2 };
533 static bool isVSplat(SDValue N, APInt &Imm, bool IsLittleEndian) {
553 static bool isVectorAllOnes(SDValue N) {
577 static bool isBitwiseInverse(SDValue N, SDValue OfNode) {
577 static bool isBitwiseInverse(SDValue N, SDValue OfNode) {
596 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
607 SDValue Op0 = N->getOperand(0);
608 SDValue Op1 = N->getOperand(1);
611 SDValue Op0Op0 = Op0->getOperand(0);
612 SDValue Op0Op1 = Op0->getOperand(1);
613 SDValue Op1Op0 = Op1->getOperand(0);
614 SDValue Op1Op1 = Op1->getOperand(1);
617 SDValue IfSet, IfClr, Cond;
792 static SDValue genConstMult(SDValue X, APInt C, const SDLoc &DL, EVT VT,
792 static SDValue genConstMult(SDValue X, APInt C, const SDLoc &DL, EVT VT,
816 SDValue Op0 = genConstMult(X, Floor, DL, VT, ShiftTy, DAG);
817 SDValue Op1 = genConstMult(X, C - Floor, DL, VT, ShiftTy, DAG);
823 SDValue Op0 = genConstMult(X, Ceil, DL, VT, ShiftTy, DAG);
824 SDValue Op1 = genConstMult(X, Ceil - C, DL, VT, ShiftTy, DAG);
828 static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
844 static SDValue performDSPShiftCombine(unsigned Opc, SDNode *N, EVT Ty,
869 static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
892 static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
898 SDValue Op0 = N->getOperand(0);
899 SDValue Op1 = N->getOperand(1);
906 SDValue Op0Op0 = Op0->getOperand(0);
922 SDValue Ops[] = { Op0Op0->getOperand(0), Op0Op0->getOperand(1),
938 static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
967 static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG) {
980 static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG) {
984 SDValue SetCC = N->getOperand(0);
997 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
1005 SDValue Op0 = N->getOperand(0);
1006 SDValue Op1 = N->getOperand(1);
1007 SDValue NotOp;
1024 SDValue
1027 SDValue Val;
1162 getOpndList(SmallVectorImpl<SDValue> &Ops,
1163 std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
1165 bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
1166 SDValue Chain) const {
1173 SDValue MipsSETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1173 SDValue MipsSETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1181 SDValue Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
1185 SDValue Lo = DAG.getLoad(MVT::i32, DL, Chain, Ptr, MachinePointerInfo(),
1190 SDValue Hi = DAG.getLoad(
1197 SDValue BP = DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1198 SDValue Ops[2] = {BP, Hi.getValue(1)};
1202 SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1202 SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1210 SDValue Val = Nd.getValue(), Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
1212 SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
1214 SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
1232 SDValue MipsSETargetLowering::lowerBITCAST(SDValue Op,
1232 SDValue MipsSETargetLowering::lowerBITCAST(SDValue Op,
1240 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32,
1242 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32,
1249 SDValue Lo =
1252 SDValue Hi =
1262 SDValue MipsSETargetLowering::lowerMulDiv(SDValue Op, unsigned NewOpc,
1262 SDValue MipsSETargetLowering::lowerMulDiv(SDValue Op, unsigned NewOpc,
1270 SDValue Mult = DAG.getNode(NewOpc, DL, MVT::Untyped,
1272 SDValue Lo, Hi;
1282 SDValue Vals[] = { Lo, Hi };
1286 static SDValue initAccumulator(SDValue In, const SDLoc &DL, SelectionDAG &DAG) {
1286 static SDValue initAccumulator(SDValue In, const SDLoc &DL, SelectionDAG &DAG) {
1287 SDValue InLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
1289 SDValue InHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
1294 static SDValue extractLOHI(SDValue Op, const SDLoc &DL, SelectionDAG &DAG) {
1294 static SDValue extractLOHI(SDValue Op, const SDLoc &DL, SelectionDAG &DAG) {
1295 SDValue Lo = DAG.getNode(MipsISD::MFLO, DL, MVT::i32, Op);
1296 SDValue Hi = DAG.getNode(MipsISD::MFHI, DL, MVT::i32, Op);
1312 static SDValue lowerDSPIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1312 static SDValue lowerDSPIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1315 SmallVector<SDValue, 3> Ops;
1326 SDValue Opnd = Op->getOperand(++OpNo), In64;
1349 SDValue Val = DAG.getNode(Opc, DL, ResTys, Ops);
1350 SDValue Out = (ResTys[0] == MVT::Untyped) ? extractLOHI(Val, DL, DAG) : Val;
1356 SDValue Vals[] = { Out, SDValue(Val.getNode(), 1) };
1361 static SDValue lowerMSACopyIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1361 static SDValue lowerMSACopyIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1363 SDValue Vec = Op->getOperand(1);
1364 SDValue Idx = Op->getOperand(2);
1368 SDValue Result = DAG.getNode(Opc, DL, ResTy, Vec, Idx,
1374 static SDValue lowerMSASplatZExt(SDValue Op, unsigned OpNr, SelectionDAG &DAG) {
1374 static SDValue lowerMSASplatZExt(SDValue Op, unsigned OpNr, SelectionDAG &DAG) {
1383 SDValue LaneA = Op->getOperand(OpNr);
1384 SDValue LaneB;
1403 SDValue Ops[16] = { LaneA, LaneB, LaneA, LaneB, LaneA, LaneB, LaneA, LaneB,
1406 SDValue Result = DAG.getBuildVector(
1410 SDValue One = DAG.getConstant(1, DL, ViaVecTy);
1418 static SDValue lowerMSASplatImm(SDValue Op, unsigned ImmOp, SelectionDAG &DAG,
1418 static SDValue lowerMSASplatImm(SDValue Op, unsigned ImmOp, SelectionDAG &DAG,
1427 static SDValue getBuildVectorSplat(EVT VecTy, SDValue SplatValue,
1427 static SDValue getBuildVectorSplat(EVT VecTy, SDValue SplatValue,
1430 SDValue SplatValueA = SplatValue;
1431 SDValue SplatValueB = SplatValue;
1449 SDValue Ops[16] = { SplatValueA, SplatValueB, SplatValueA, SplatValueB,
1454 SDValue Result = DAG.getBuildVector(
1463 static SDValue lowerMSABinaryBitImmIntr(SDValue Op, SelectionDAG &DAG,
1463 static SDValue lowerMSABinaryBitImmIntr(SDValue Op, SelectionDAG &DAG,
1464 unsigned Opc, SDValue Imm,
1467 SDValue Exp2Imm;
1476 SDValue BitImmHiOp = DAG.getConstant(BitImm.lshr(32).trunc(32), DL,
1478 SDValue BitImmLoOp = DAG.getConstant(BitImm.trunc(32), DL, MVT::i32);
1507 static SDValue truncateVecElts(SDValue Op, SelectionDAG &DAG) {
1507 static SDValue truncateVecElts(SDValue Op, SelectionDAG &DAG) {
1510 SDValue Vec = Op->getOperand(2);
1513 SDValue ConstValue = DAG.getConstant(Vec.getScalarValueSizeInBits() - 1,
1515 SDValue SplatVec = getBuildVectorSplat(ResTy, ConstValue, BigEndian, DAG);
1520 static SDValue lowerMSABitClear(SDValue Op, SelectionDAG &DAG) {
1520 static SDValue lowerMSABitClear(SDValue Op, SelectionDAG &DAG) {
1523 SDValue One = DAG.getConstant(1, DL, ResTy);
1524 SDValue Bit = DAG.getNode(ISD::SHL, DL, ResTy, One, truncateVecElts(Op, DAG));
1530 static SDValue lowerMSABitClearImm(SDValue Op, SelectionDAG &DAG) {
1530 static SDValue lowerMSABitClearImm(SDValue Op, SelectionDAG &DAG) {
1535 SDValue BitMask = DAG.getConstant(~BitImm, DL, ResTy);
1540 SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
1540 SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
1656 SDValue One = DAG.getConstant(1, DL, VecTy);
1692 SDValue One = DAG.getConstant(1, DL, VecTy);
1882 SmallVector<SDValue, 16> Ops(ResTy.getVectorNumElements(),
2086 SDValue Res = DAG.getNode(ISD::OR, DL, Op->getValueType(0),
2091 SDValue Res = DAG.getNode(ISD::OR, DL, Op->getValueType(0),
2293 static SDValue lowerMSALoadIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr,
2293 static SDValue lowerMSALoadIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr,
2296 SDValue ChainIn = Op->getOperand(0);
2297 SDValue Address = Op->getOperand(2);
2298 SDValue Offset = Op->getOperand(3);
2313 SDValue MipsSETargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
2313 SDValue MipsSETargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
2367 static SDValue lowerMSAStoreIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr,
2367 static SDValue lowerMSAStoreIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr,
2370 SDValue ChainIn = Op->getOperand(0);
2371 SDValue Value = Op->getOperand(2);
2372 SDValue Address = Op->getOperand(3);
2373 SDValue Offset = Op->getOperand(4);
2388 SDValue MipsSETargetLowering::lowerINTRINSIC_VOID(SDValue Op,
2388 SDValue MipsSETargetLowering::lowerINTRINSIC_VOID(SDValue Op,
2409 SDValue MipsSETargetLowering::
2410 lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
2413 SDValue Op0 = Op->getOperand(0);
2420 SDValue Op1 = Op->getOperand(1);
2429 static bool isConstantOrUndef(const SDValue Op) {
2459 SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
2459 SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
2505 SDValue Result = DAG.getConstant(SplatValue, DL, ViaVecTy);
2523 SDValue Vector = DAG.getUNDEF(ResTy);
2553 static SDValue lowerVECTOR_SHUFFLE_SHF(SDValue Op, EVT ResTy,
2553 static SDValue lowerVECTOR_SHUFFLE_SHF(SDValue Op, EVT ResTy,
2634 static bool isVECTOR_SHUFFLE_SPLATI(SDValue Op, EVT ResTy,
2666 static SDValue lowerVECTOR_SHUFFLE_ILVEV(SDValue Op, EVT ResTy,
2666 static SDValue lowerVECTOR_SHUFFLE_ILVEV(SDValue Op, EVT ResTy,
2671 SDValue Wt;
2672 SDValue Ws;
2712 static SDValue lowerVECTOR_SHUFFLE_ILVOD(SDValue Op, EVT ResTy,
2712 static SDValue lowerVECTOR_SHUFFLE_ILVOD(SDValue Op, EVT ResTy,
2717 SDValue Wt;
2718 SDValue Ws;
2759 static SDValue lowerVECTOR_SHUFFLE_ILVR(SDValue Op, EVT ResTy,
2759 static SDValue lowerVECTOR_SHUFFLE_ILVR(SDValue Op, EVT ResTy,
2764 SDValue Wt;
2765 SDValue Ws;
2806 static SDValue lowerVECTOR_SHUFFLE_ILVL(SDValue Op, EVT ResTy,
2806 static SDValue lowerVECTOR_SHUFFLE_ILVL(SDValue Op, EVT ResTy,
2812 SDValue Wt;
2813 SDValue Ws;
2854 static SDValue lowerVECTOR_SHUFFLE_PCKEV(SDValue Op, EVT ResTy,
2854 static SDValue lowerVECTOR_SHUFFLE_PCKEV(SDValue Op, EVT ResTy,
2859 SDValue Wt;
2860 SDValue Ws;
2897 static SDValue lowerVECTOR_SHUFFLE_PCKOD(SDValue Op, EVT ResTy,
2897 static SDValue lowerVECTOR_SHUFFLE_PCKOD(SDValue Op, EVT ResTy,
2902 SDValue Wt;
2903 SDValue Ws;
2933 static SDValue lowerVECTOR_SHUFFLE_VSHF(SDValue Op, EVT ResTy,
2933 static SDValue lowerVECTOR_SHUFFLE_VSHF(SDValue Op, EVT ResTy,
2936 SmallVector<SDValue, 16> Ops;
2937 SDValue Op0;
2938 SDValue Op1;
2960 SDValue MaskVec = DAG.getBuildVector(MaskVecTy, DL, Ops);
2984 SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
2984 SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
3002 SDValue Result;
lib/Target/Mips/MipsSEISelLowering.h 48 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
48 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
50 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
68 getOpndList(SmallVectorImpl<SDValue> &Ops,
69 std::deque<std::pair<unsigned, SDValue>> &RegsToPass,
71 bool IsCallReloc, CallLoweringInfo &CLI, SDValue Callee,
72 SDValue Chain) const override;
74 SDValue lowerLOAD(SDValue Op, SelectionDAG &DAG) const;
74 SDValue lowerLOAD(SDValue Op, SelectionDAG &DAG) const;
75 SDValue lowerSTORE(SDValue Op, SelectionDAG &DAG) const;
75 SDValue lowerSTORE(SDValue Op, SelectionDAG &DAG) const;
76 SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
76 SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
78 SDValue lowerMulDiv(SDValue Op, unsigned NewOpc, bool HasLo, bool HasHi,
78 SDValue lowerMulDiv(SDValue Op, unsigned NewOpc, bool HasLo, bool HasHi,
81 SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
81 SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
83 SDValue lowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
83 SDValue lowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
84 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
84 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
85 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
85 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
88 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
88 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
89 SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
89 SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp 526 SDValue Val = CurDAG->getTargetConstantFP(
605 SDValue Vector = N->getOperand(0);
638 SDValue Source = Vector;
731 SDValue Wrapper = N->getOperand(1);
732 SDValue GlobalVal = Wrapper.getOperand(0);
738 SDValue Src = N->getOperand(0);
909 SDValue Chain = N->getOperand(0);
910 SDValue N1 = N->getOperand(1);
911 SDValue Addr;
912 SDValue Offset, Base;
923 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
936 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
955 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
974 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
993 SDValue Chain = N->getOperand(0);
994 SDValue Op1 = N->getOperand(1);
995 SDValue Addr, Offset, Base;
1092 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1119 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1168 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1217 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1232 SDValue Chain = N->getOperand(0);
1233 SDValue Op1;
1265 SDValue Base, Offset, Addr;
1363 SDValue Ops[] = { Addr, Chain };
1510 SDValue Ops[] = {Base, Offset, Chain};
1656 SDValue Ops[] = { Op1, Chain };
1688 SDValue Res(LD, i);
1689 SDValue OrigVal(N, i);
1766 SDValue Chain = ST->getChain();
1767 SDValue Value = PlainStore ? PlainStore->getValue() : AtomicStore->getVal();
1768 SDValue BasePtr = ST->getBasePtr();
1769 SDValue Addr;
1770 SDValue Offset, Base;
1782 SDValue Ops[] = {Value,
1800 SDValue Ops[] = {Value,
1826 SDValue Ops[] = {Value,
1850 SDValue Ops[] = {Value,
1871 SDValue Chain = N->getOperand(0);
1872 SDValue Op1 = N->getOperand(1);
1873 SDValue Addr, Offset, Base;
1910 SmallVector<SDValue, 12> StOps;
1911 SDValue N2;
2095 SDValue Chain = Node->getOperand(0);
2096 SDValue Offset = Node->getOperand(2);
2097 SDValue Flag = Node->getOperand(3);
2162 SmallVector<SDValue, 2> Ops;
2173 SDValue Chain = N->getOperand(0);
2174 SDValue Offset = N->getOperand(1);
2195 SmallVector<SDValue, 6> Ops;
2243 SDValue Chain = N->getOperand(0);
2244 SDValue Param = N->getOperand(1);
2246 SDValue Offset = N->getOperand(2);
2249 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2270 SmallVector<SDValue, 8> Ops;
2317 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
2326 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
2857 SmallVector<SDValue, 8> Ops(N->op_begin() + 1, N->op_end());
3366 SmallVector<SDValue, 8> Ops(N->op_begin() + 1, N->op_end());
3378 SDValue LHS = N->getOperand(0);
3379 SDValue RHS = N->getOperand(1);
3380 SDValue Len;
3381 SDValue Start;
3382 SDValue Val;
3451 SDValue AndLHS = LHS->getOperand(0);
3452 SDValue AndRHS = LHS->getOperand(1);
3504 SDValue ShlRHS = LHS->getOperand(1);
3512 SDValue ShrRHS = RHS;
3572 SDValue Ops[] = {
3582 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
3582 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
3605 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3605 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3605 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3608 SDValue base = Addr.getOperand(0);
3620 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
3621 SDValue &Base, SDValue &Offset) {
3621 SDValue &Base, SDValue &Offset) {
3626 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
3627 SDValue &Base, SDValue &Offset) {
3627 SDValue &Base, SDValue &Offset) {
3633 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3633 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3633 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3663 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
3664 SDValue &Base, SDValue &Offset) {
3664 SDValue &Base, SDValue &Offset) {
3669 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
3670 SDValue &Base, SDValue &Offset) {
3670 SDValue &Base, SDValue &Offset) {
3692 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
3692 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
3693 SDValue Op0, Op1;
lib/Target/NVPTX/NVPTXISelDAGToDAG.h 51 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
53 std::vector<SDValue> &OutOps) override;
78 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
83 bool SelectDirectAddr(SDValue N, SDValue &Address);
83 bool SelectDirectAddr(SDValue N, SDValue &Address);
85 bool SelectADDRri_imp(SDNode *OpNode, SDValue Addr, SDValue &Base,
85 bool SelectADDRri_imp(SDNode *OpNode, SDValue Addr, SDValue &Base,
86 SDValue &Offset, MVT mvt);
87 bool SelectADDRri(SDNode *OpNode, SDValue Addr, SDValue &Base,
87 bool SelectADDRri(SDNode *OpNode, SDValue Addr, SDValue &Base,
88 SDValue &Offset);
89 bool SelectADDRri64(SDNode *OpNode, SDValue Addr, SDValue &Base,
89 bool SelectADDRri64(SDNode *OpNode, SDValue Addr, SDValue &Base,
90 SDValue &Offset);
91 bool SelectADDRsi_imp(SDNode *OpNode, SDValue Addr, SDValue &Base,
91 bool SelectADDRsi_imp(SDNode *OpNode, SDValue Addr, SDValue &Base,
92 SDValue &Offset, MVT mvt);
93 bool SelectADDRsi(SDNode *OpNode, SDValue Addr, SDValue &Base,
93 bool SelectADDRsi(SDNode *OpNode, SDValue Addr, SDValue &Base,
94 SDValue &Offset);
95 bool SelectADDRsi64(SDNode *OpNode, SDValue Addr, SDValue &Base,
95 bool SelectADDRsi64(SDNode *OpNode, SDValue Addr, SDValue &Base,
96 SDValue &Offset);
lib/Target/NVPTX/NVPTXISelLowering.cpp 1200 SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
1200 SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
1249 SDValue
1250 NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
1370 unsigned NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
1422 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1423 SmallVectorImpl<SDValue> &InVals) const {
1427 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1429 SDValue Chain = CLI.Chain;
1430 SDValue Callee = CLI.Callee;
1442 SDValue tempChain = Chain;
1444 SDValue InFlag = Chain.getValue(1);
1474 SDValue DeclareParamOps[] = {
1489 SDValue DeclareScalarParamOps[] = {
1507 SmallVector<SDValue, 6> StoreOperands;
1518 SDValue StVal = OutVals[OIdx];
1597 SDValue DeclareParamOps[] = {Chain, DAG.getConstant(ArgAlign, dl, MVT::i32),
1608 SDValue srcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, OutVals[OIdx],
1610 SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr,
1616 SDValue CopyParamOps[] = { Chain,
1652 SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
1661 SDValue DeclareRetOps[] = { Chain,
1700 SDValue ProtoOps[] = {
1708 SDValue PrintCallOps[] = {
1721 SDValue CallVoidOps[] = { Chain, Callee, InFlag };
1727 SDValue CallArgBeginOps[] = { Chain, InFlag };
1739 SDValue CallArgOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
1745 SDValue CallArgEndOps[] = { Chain,
1753 SDValue PrototypeOps[] = { Chain,
1760 SmallVector<SDValue, 16> ProxyRegOps;
1824 SDValue LoadOperands[] = {
1827 SDValue RetVal = DAG.getMemIntrinsicNode(
1863 SDValue Ret = DAG.getNode(
1888 SDValue
1889 NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
1892 SmallVector<SDValue, 8> Ops;
1895 SDValue SubOp = Node->getOperand(i);
1917 SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1917 SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1929 SDValue Const =
1934 SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
1934 SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
1936 SDValue Index = Op->getOperand(1);
1942 SDValue Vector = Op->getOperand(0);
1948 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,
1950 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,
1961 SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
1961 SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
1969 SDValue ShOpLo = Op.getOperand(0);
1970 SDValue ShOpHi = Op.getOperand(1);
1971 SDValue ShAmt = Op.getOperand(2);
1980 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
1981 SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi,
1984 SDValue Ops[2] = { Lo, Hi };
1996 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
1999 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
2000 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
2002 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
2003 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2004 SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
2006 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
2009 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
2010 SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
2012 SDValue Ops[2] = { Lo, Hi };
2022 SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
2022 SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
2030 SDValue ShOpLo = Op.getOperand(0);
2031 SDValue ShOpHi = Op.getOperand(1);
2032 SDValue ShAmt = Op.getOperand(2);
2040 SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi,
2042 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
2044 SDValue Ops[2] = { Lo, Hi };
2056 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
2059 SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
2060 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
2062 SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
2063 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2064 SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
2066 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
2069 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
2070 SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
2072 SDValue Ops[2] = { Lo, Hi };
2077 SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2077 SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2096 SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
2096 SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
2099 SDValue A = Op.getOperand(0);
2102 SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);
2105 SDValue Bitcast = DAG.getNode(ISD::BITCAST, SL, MVT::i32, A);
2107 SDValue Sign = DAG.getNode(ISD::AND, SL, MVT::i32, Bitcast,
2110 SDValue PointFiveWithSignRaw =
2113 SDValue PointFiveWithSign =
2115 SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, A, PointFiveWithSign);
2116 SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);
2120 SDValue IsLarge =
2126 SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,
2128 SDValue RoundedAForSmallA = DAG.getNode(ISD::FTRUNC, SL, VT, A);
2137 SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
2137 SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
2140 SDValue A = Op.getOperand(0);
2143 SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);
2146 SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, AbsA,
2148 SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);
2152 SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,
2163 SDValue IsLarge =
2171 SDValue
2172 NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
2208 SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const {
2208 SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const {
2209 SDValue Op0 = Op->getOperand(0);
2210 SDValue Op1 = Op->getOperand(1);
2211 SDValue Op2 = Op->getOperand(2);
2218 SDValue Select = DAG.getNode(ISD::SELECT, DL, MVT::i32, Op0, Op1, Op2);
2219 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);
2224 SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
2224 SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
2235 SDValue Ops[2];
2248 SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
2248 SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
2255 SDValue newLD = DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),
2258 SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
2262 SDValue Ops[] = { result, LD->getChain() };
2266 SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
2266 SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
2286 SDValue
2287 NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
2289 SDValue Val = N->getOperand(1);
2365 SmallVector<SDValue, 8> Ops;
2374 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Val,
2376 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Val,
2378 SDValue V2 = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f16, E0, E1);
2384 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
2395 SDValue NewSt =
2410 SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
2410 SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
2414 SDValue Tmp1 = ST->getChain();
2415 SDValue Tmp2 = ST->getBasePtr();
2416 SDValue Tmp3 = ST->getValue();
2419 SDValue Result =
2425 SDValue
2462 SDValue NVPTXTargetLowering::LowerFormalArguments(
2463 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2465 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2474 SDValue Root = DAG.getRoot();
2475 std::vector<SDValue> OutChains;
2563 SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
2586 SDValue VecAddr =
2591 SDValue P =
2599 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LoadVT, P,
2638 SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
2639 SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
2659 SDValue
2660 NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2663 const SmallVectorImpl<SDValue> &OutVals,
2688 SmallVector<SDValue, 6> StoreOperands;
2697 SDValue RetVal = OutVals[i];
2745 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2745 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
4342 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
4342 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
4342 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
4448 static SDValue PerformADDCombine(SDNode *N,
4452 SDValue N0 = N->getOperand(0);
4453 SDValue N1 = N->getOperand(1);
4456 if (SDValue Result =
4464 static SDValue PerformANDCombine(SDNode *N,
4471 SDValue Val = N->getOperand(0);
4472 SDValue Mask = N->getOperand(1);
4478 SDValue AExt;
4539 static SDValue PerformREMCombine(SDNode *N,
4554 const SDValue &Num = N->getOperand(0);
4555 const SDValue &Den = N->getOperand(1);
4579 static bool IsMulWideOperandDemotable(SDValue Op,
4606 static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
4606 static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
4642 static SDValue TryMULWIDECombine(SDNode *N,
4651 SDValue LHS = N->getOperand(0);
4652 SDValue RHS = N->getOperand(1);
4693 SDValue TruncLHS =
4695 SDValue TruncRHS =
4709 static SDValue PerformMULCombine(SDNode *N,
4714 if (SDValue Ret = TryMULWIDECombine(N, DCI))
4722 static SDValue PerformSHLCombine(SDNode *N,
4727 if (SDValue Ret = TryMULWIDECombine(N, DCI))
4734 static SDValue PerformSETCCCombine(SDNode *N,
4737 SDValue A = N->getOperand(0);
4738 SDValue B = N->getOperand(1);
4748 SDValue CCNode = DCI.DAG.getNode(NVPTXISD::SETP_F16X2, DL,
4755 SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
4780 SmallVectorImpl<SDValue> &Results) {
4869 SmallVector<SDValue, 8> OtherOps(N->op_begin(), N->op_end());
4875 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
4879 SmallVector<SDValue, 8> ScalarRes;
4884 SDValue SubVector = NewLD.getValue(i);
4885 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SubVector,
4887 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SubVector,
4894 SDValue Res = NewLD.getValue(i);
4901 SDValue LoadChain = NewLD.getValue(NumElts);
4903 SDValue BuildVec = DAG.getBuildVector(ResVT, DL, ScalarRes);
4910 SmallVectorImpl<SDValue> &Results) {
4911 SDValue Chain = N->getOperand(0);
4912 SDValue Intrin = N->getOperand(1);
4988 SmallVector<SDValue, 8> OtherOps;
4999 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
5003 SmallVector<SDValue, 4> ScalarRes;
5006 SDValue Res = NewLD.getValue(i);
5013 SDValue LoadChain = NewLD.getValue(NumElts);
5015 SDValue BuildVec =
5026 SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
5035 SDValue NewLD =
5048 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
lib/Target/NVPTX/NVPTXISelLowering.h 444 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
444 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
446 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
446 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
483 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
483 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
487 SmallVectorImpl<SDValue> &InVals) const override;
489 SDValue LowerCall(CallLoweringInfo &CLI,
490 SmallVectorImpl<SDValue> &InVals) const override;
497 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
497 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
499 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
502 void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
503 std::vector<SDValue> &Ops,
532 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
532 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
553 SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
555 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
555 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
556 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
556 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
557 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
557 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
559 SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const;
559 SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const;
560 SDValue LowerFROUND32(SDValue Op, SelectionDAG &DAG) const;
560 SDValue LowerFROUND32(SDValue Op, SelectionDAG &DAG) const;
561 SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const;
561 SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const;
563 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
563 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
564 SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const;
564 SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const;
566 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
566 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
567 SDValue LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const;
567 SDValue LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const;
568 SDValue LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const;
568 SDValue LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const;
570 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
570 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
571 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
571 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
573 SDValue LowerSelect(SDValue Op, SelectionDAG &DAG) const;
573 SDValue LowerSelect(SDValue Op, SelectionDAG &DAG) const;
575 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
577 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
579 unsigned getArgumentAlignment(SDValue Callee, ImmutableCallSite CS, Type *Ty,
lib/Target/PowerPC/PPCISelDAGToDAG.cpp 167 inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) {
173 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
179 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) {
184 inline SDValue getSmallIPtrImm(unsigned Imm, const SDLoc &dl) {
218 SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
218 SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
218 SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
224 bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
224 bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
240 bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
240 bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
240 bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
250 bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
250 bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
250 bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
260 bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
260 bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
260 bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
266 bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
266 bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
266 bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
274 bool SelectAddrImm(SDValue N, SDValue &Disp,
274 bool SelectAddrImm(SDValue N, SDValue &Disp,
275 SDValue &Base) {
282 bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
282 bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
282 bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
289 bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
289 bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
289 bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
294 bool SelectAddr(SDValue N, SDValue &Base) {
294 bool SelectAddr(SDValue N, SDValue &Base) {
304 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
306 std::vector<SDValue> &OutOps) override {
323 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
324 SDValue NewOp =
351 SDValue combineToCMPB(SDNode *N);
352 void foldBoolExts(SDValue &Res, SDNode *&N);
505 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
511 static bool isInt64Immediate(SDValue N, uint64_t &Imm) {
516 const SDValue &DestMBB) {
576 SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
629 SDValue Base = ST->getBasePtr();
632 SDValue Offset = ST->getOffset();
661 SDValue Chain = ST->getChain();
663 SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1),
672 SDValue Base = LD->getBasePtr();
675 SDValue Offset = LD->getOffset();
703 SDValue Chain = LD->getChain();
705 SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain};
715 SDValue Op0 = N->getOperand(0);
716 SDValue Op1 = N->getOperand(1);
778 SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
925 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
946 SDValue Ops[] =
1027 SDValue Val = SDValue(selectI64ImmDirect(CurDAG, dl, MatImm), 0);
1096 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
1107 SDValue V;
1123 ValueBit(SDValue V, unsigned I, Kind K = Variable)
1136 SDValue getValue() const {
1149 SDValue V;
1163 BitGroup(SDValue V, unsigned R, unsigned S, unsigned E)
1174 SDValue V;
1208 DenseMap<SDValue, std::unique_ptr<ValueBitsMemoizedValue>>;
1300 SDValue LastVal = SDValue();
1463 SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue();
1468 SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue();
1702 SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
1721 SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
1721 SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
1726 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1727 SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
1729 SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
1735 SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
1735 SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
1740 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1741 SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
1749 void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
1801 SDValue VRot;
1803 SDValue Ops[] =
1812 SDValue ANDIVal, ANDISVal;
1820 SDValue TotalVal;
1846 SDValue Res;
1860 SDValue Ops[] =
1880 SDValue Ops[] =
1886 SDValue Ops[] =
1905 SDValue ANDIVal, ANDISVal;
1945 SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
1945 SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
1961 SDValue Ops[] =
1969 SDValue Ops[] =
1976 SDValue Ops[] =
1983 SDValue Ops[] =
2009 SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2009 SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2009 SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2025 SDValue Ops[] =
2033 SDValue Ops[] =
2057 void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2166 SDValue VRot;
2177 SDValue TotalVal;
2182 SDValue ANDIVal, ANDISVal;
2225 SDValue Res;
2331 SDValue ANDIVal, ANDISVal;
2349 SDValue MaskVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2393 DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots;
2478 SDValue computeLogicOpInGPR(SDValue LogicOp);
2478 SDValue computeLogicOpInGPR(SDValue LogicOp);
2479 SDValue signExtendInputIfNeeded(SDValue Input);
2479 SDValue signExtendInputIfNeeded(SDValue Input);
2480 SDValue zeroExtendInputIfNeeded(SDValue Input);
2480 SDValue zeroExtendInputIfNeeded(SDValue Input);
2481 SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2481 SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2482 SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2482 SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2484 SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2484 SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2484 SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2486 SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2486 SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2486 SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2488 SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2488 SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2488 SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2490 SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2490 SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2490 SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2492 SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2492 SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2534 SDValue WideRes;
2559 SDValue ConvOp = WideRes;
2575 SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0));
2582 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
2583 SDValue LHS = LoweredLogical.getOperand(0);
2584 SDValue RHS = LoweredLogical.getOperand(1);
2585 SDValue WideOp;
2586 SDValue OpToConvToRecForm;
2614 std::vector<SDValue> Ops;
2634 SDValue SRIdxVal =
2636 SDValue CRBit =
2651 SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
2651 SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
2657 SDValue LHS, RHS;
2671 SDValue InputOp = Operand.getOperand(0);
2719 SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
2719 SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
2752 SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
2752 SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
2788 SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
2788 SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
2795 SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0);
2796 SDValue SubRegIdx =
2806 SDValue SubRegIdx =
2814 SDValue
2815 IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2819 SDValue ToExtend;
2833 SDValue Neg =
2840 SDValue Addi =
2865 SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
2888 SDValue
2889 IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
2889 IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
2903 SDValue Xor = IsRHSZero ? LHS :
2905 SDValue Clz =
2907 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
2915 SDValue Xor = IsRHSZero ? LHS :
2917 SDValue Clz =
2919 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
2921 SDValue Shift =
2953 SDValue Sub =
2955 SDValue Shift =
2977 SDValue Neg =
3002 SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3013 SDValue SUBFNode =
3030 SDValue Subtract =
3032 SDValue SrdiNode =
3050 SDValue Subtract =
3061 SDValue
3062 IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3062 IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3079 SDValue CountInput = IsRHSZero ? LHS :
3081 SDValue Cntlzw =
3083 SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3085 SDValue Slwi =
3097 SDValue Xor = IsRHSZero ? LHS :
3099 SDValue Clz =
3101 SDValue ShiftOps[] =
3103 SDValue Shift =
3105 SDValue Xori =
3134 SDValue SUBFNode =
3137 SDValue Srdi =
3156 SDValue Neg =
3187 SDValue SUBFNode =
3203 SDValue Subtract =
3205 SDValue Shift =
3223 SDValue Subtract =
3233 SDValue
3234 IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3234 IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3248 SDValue Xor = IsRHSZero ? LHS :
3250 SDValue Clz =
3261 SDValue Xor = IsRHSZero ? LHS :
3263 SDValue AC =
3288 SDValue ShiftL =
3292 SDValue ShiftR =
3295 SDValue SubtractCarry =
3309 SDValue Addi =
3312 SDValue Nor =
3335 SDValue SRADINode =
3338 SDValue SRDINode =
3342 SDValue SUBFC8Carry =
3345 SDValue ADDE8Node =
3359 SDValue SUBFC8Carry =
3362 SDValue SUBFE8Node =
3376 SDValue SubtractCarry =
3379 SDValue ExtSub =
3390 SDValue
3391 IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3391 IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3407 SDValue AddInput = IsRHSZero ? LHS :
3409 SDValue Addic =
3420 SDValue Xor = IsRHSZero ? LHS :
3422 SDValue SC =
3447 SDValue ShiftR =
3450 SDValue ShiftL =
3454 SDValue SubtractCarry =
3457 SDValue Adde =
3470 SDValue Add =
3473 SDValue Nor =
3495 SDValue SRADINode =
3498 SDValue SRDINode =
3502 SDValue SUBFC8Carry =
3505 SDValue ADDE8Node =
3508 SDValue XORI8Node =
3522 SDValue SubtractCarry =
3525 SDValue ExtSub =
3539 SDValue SubCarry =
3552 static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) {
3576 SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
3576 SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
3587 SDValue LHS = Compare.getOperand(0);
3588 SDValue RHS = Compare.getOperand(1);
3678 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3678 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3678 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3707 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
3752 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
4020 SDValue Op = N->getOperand(0);
4025 SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
4032 SDValue AD =
4039 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4045 SDValue T =
4048 SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
4055 SDValue Op = N->getOperand(0);
4078 SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
4080 SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
4082 SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4088 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4098 SDValue LHS = N->getOperand(0);
4099 SDValue RHS = N->getOperand(1);
4116 SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
4131 SDValue CCReg = SelectCC(LHS, RHS, CC, dl);
4132 SDValue IntCR;
4141 SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32);
4143 SDValue InFlag(nullptr, 0); // Null incoming flag value.
4150 SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
4158 SDValue Tmp =
4169 SDValue AddrOp;
4212 SDValue LHS = N->getOperand(0);
4213 SDValue RHS = N->getOperand(1);
4214 SDValue TrueRes = N->getOperand(2);
4215 SDValue FalseRes = N->getOperand(3);
4238 SDValue SetOrSelCC = InnerIsSel ? FalseRes : FalseRes.getOperand(0);
4254 SDValue InnerLHS = SetOrSelCC.getOperand(0);
4255 SDValue InnerRHS = SetOrSelCC.getOperand(1);
4404 SDValue Op = N->getOperand(1);
4426 SDValue InFlag = N->getOperand(1);
4438 SDValue N0 = N->getOperand(0);
4439 SDValue ShiftAmt =
4485 SDValue Offset = LD->getOffset();
4516 SDValue Chain = LD->getChain();
4517 SDValue Base = LD->getBasePtr();
4518 SDValue Ops[] = { Offset, Base, Chain };
4556 SDValue Chain = LD->getChain();
4557 SDValue Base = LD->getBasePtr();
4558 SDValue Ops[] = { Base, Offset, Chain };
4576 SDValue Val = N->getOperand(0).getOperand(0);
4577 SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl),
4587 SDValue Val = N->getOperand(0);
4588 SDValue Ops[] = { Val, getI32Imm(0, dl), getI32Imm(MB, dl),
4596 SDValue Val = N->getOperand(0);
4601 auto Op0 = Val.getOperand(0);
4608 SDValue IDVal (ImDef, 0);
4629 SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) };
4638 SDValue Val = N->getOperand(0);
4641 SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) };
4671 SDValue Ops[] = { N->getOperand(0).getOperand(0),
4714 SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)};
4737 SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)};
4759 SDValue Ops[] = { N->getOperand(0).getOperand(0),
4773 SDValue Ops[] = { N->getOperand(0).getOperand(0),
4794 SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue,
4798 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
4799 SDValue SRIdxVal =
4822 SDValue LHS = N->getOperand(0);
4823 SDValue RHS = N->getOperand(1);
4833 SDValue GenCC =
4860 SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl);
4876 SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg);
4878 SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1,
4880 SDValue C = Inv ? NotCCBit : CCBit,
4883 SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1,
4885 SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1,
4929 SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3),
4939 SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1),
4953 SDValue Base, Offset;
4959 SDValue Chain = LD->getChain();
4960 SDValue Ops[] = { Base, Offset, Chain };
4978 SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl,
4980 SDValue Ops[] = { Op1, Op2, DMV };
4989 SDValue Ops[] = { N->getOperand(1), N->getOperand(0) };
5007 SDValue Pred = getI32Imm(PCC, dl);
5008 SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3),
5039 SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1,
5050 SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl);
5051 SDValue Ops[] = { getI32Imm(PCC, dl), CondCode,
5058 SDValue Chain = N->getOperand(0);
5059 SDValue Target = N->getOperand(1);
5091 SDValue GA = TocEntry->getOperand(0);
5092 SDValue TocBase = TocEntry->getOperand(1);
5130 SDValue GA = N->getOperand(0);
5131 SDValue TOCbase = N->getOperand(1);
5199 SDValue EltVal = getI32Imm(Elt >> 1, dl);
5201 SDValue TmpVal = SDValue(Tmp, 0);
5211 SDValue EltVal = getI32Imm(Elt - 16, dl);
5225 SDValue EltVal = getI32Imm(Elt + 16, dl);
5245 SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
5251 SDValue Res;
5261 SDValue RHS, LHS;
5291 SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1);
5345 SDValue Op = O.getOperand(0);
5352 SDValue XOR = Op.getOperand(0);
5370 SDValue XOR = Op.getOperand(0);
5384 SmallVector<SDValue, 8> Queue(1, SDValue(N, 0));
5386 SDValue V = Queue.pop_back_val();
5388 for (const SDValue &O : V.getNode()->ops()) {
5391 SDValue OLHS, ORHS;
5462 void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) {
5479 SDValue Cond = N->getOperand(0);
5480 SDValue ConstTrue =
5482 SDValue ConstFalse = CurDAG->getConstant(0, dl, VT);
5490 SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1);
5491 SDValue O0 = UserO0.getNode() == N ? Val : UserO0;
5492 SDValue O1 = UserO1.getNode() == N ? Val : UserO1;
5501 SDValue TrueRes = TryFold(ConstTrue);
5504 SDValue FalseRes = TryFold(ConstFalse);
5533 SDValue Res;
5662 SDValue Op = MachineNode->getOperand(1);
5689 SDValue Op = MachineNode->getOperand(0);
6061 static bool PeepholePPC64ZExtGather(SDValue Op32,
6233 SDValue ISR = N->getOperand(0);
6244 SDValue IDef = ISR.getOperand(0);
6252 SDValue Op32 = ISR->getOperand(1);
6316 SmallVector<SDValue, 4> Ops;
6317 for (const SDValue &V : PN->ops()) {
6320 SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) };
6432 SDValue Base = N->getOperand(FirstOp + 1);
6477 SDValue ImmOpnd = Base.getOperand(1);
6490 SDValue HBase = Base.getOperand(0);
6508 SDValue HImmOpnd = HBase.getOperand(1);
lib/Target/PowerPC/PPCISelLowering.cpp 127 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
127 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
1444 static bool isFloatingPointZero(SDValue Op) {
2104 SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
2105 SDValue OpVal(nullptr, 0);
2114 SDValue UniquedVals[4];
2252 bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
2259 bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2259 bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2260 SDValue &Index,
2281 bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base,
2281 bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base,
2282 SDValue &Index, SelectionDAG &DAG,
2368 bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
2368 bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
2369 SDValue &Base,
2467 bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2467 bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2468 SDValue &Index,
2527 SDValue LoadedVal(N, 0);
2543 bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
2544 SDValue &Offset,
2550 SDValue Ptr;
2593 SDValue Val = cast<StoreSDNode>(N)->getValue();
2662 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
2662 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
2662 static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
2666 SDValue Zero = DAG.getConstant(0, DL, PtrVT);
2668 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero);
2669 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero);
2690 SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
2691 SDValue GA) const {
2694 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT)
2698 SDValue Ops[] = { GA, Reg };
2705 SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
2705 SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
2715 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0);
2724 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(),
2729 SDValue CPIHi =
2731 SDValue CPILo =
2752 SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
2752 SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
2783 SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
2783 SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
2791 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
2800 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT,
2805 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag);
2806 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag);
2810 SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
2810 SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
2820 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset());
2833 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag);
2834 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag);
2838 SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
2838 SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
2859 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
2861 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
2863 SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64)
2866 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg);
2871 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
2872 SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
2874 SDValue GOTPtr;
2877 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
2888 SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl,
2894 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
2895 SDValue GOTPtr;
2898 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
2912 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
2913 SDValue GOTPtr;
2916 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
2925 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl,
2927 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl,
2935 SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
2935 SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
2946 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset());
2955 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT,
2961 SDValue GAHi =
2963 SDValue GALo =
2966 SDValue Ptr = LowerLabelRef(GAHi, GALo, IsPIC, DAG);
2975 SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
2975 SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3003 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
3022 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0),
3029 SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3029 SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3033 SDValue InChain = Node->getOperand(0);
3034 SDValue VAListPtr = Node->getOperand(1);
3041 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
3047 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex,
3049 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd,
3051 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex,
3059 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
3063 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
3067 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
3070 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
3074 SDValue OverflowArea =
3078 SDValue RegSaveArea =
3083 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex,
3087 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32,
3093 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea,
3102 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32,
3112 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea);
3115 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea,
3128 SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3128 SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3139 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3139 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3144 SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3144 SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3146 SDValue Chain = Op.getOperand(0);
3147 SDValue Trmp = Op.getOperand(1); // trampoline
3148 SDValue FPtr = Op.getOperand(2); // nested function
3149 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
3176 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3176 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3180 SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3180 SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3190 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
3220 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32);
3221 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32);
3222 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(),
3224 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3228 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT);
3231 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT);
3234 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT);
3239 SDValue firstStore =
3243 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1),
3247 SDValue secondStore =
3254 SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr,
3402 SDValue PPCTargetLowering::LowerFormalArguments(
3403 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3405 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3419 SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
3420 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3422 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3529 SDValue ArgValue;
3536 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32);
3537 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32);
3565 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3595 SmallVector<SDValue, 8> MemOps;
3627 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
3638 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
3639 SDValue Store =
3643 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
3657 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64);
3658 SDValue Store =
3662 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl,
3676 SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
3678 SDValue ArgVal,
3690 SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
3691 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3693 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3755 SmallVector<SDValue, 8> MemOps;
3759 SDValue ArgVal;
3809 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3826 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3832 SDValue Arg = FIN;
3834 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT);
3842 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
3843 SDValue Store;
3877 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
3878 SDValue Addr = FIN;
3880 SDValue Off = DAG.getConstant(j, dl, PtrVT);
3883 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr,
4049 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4078 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4086 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4087 SDValue Store =
4091 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
4102 SDValue PPCTargetLowering::LowerFormalArguments_Darwin(
4103 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4105 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4198 SmallVector<SDValue, 8> MemOps;
4203 SDValue ArgVal;
4245 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4254 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4256 SDValue Store =
4278 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4279 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4280 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4401 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4433 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4446 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4447 SDValue Store =
4451 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
4479 static bool isFunctionGlobalAddress(SDValue Callee);
4482 callsShareTOCBase(const Function *Caller, SDValue Callee,
4630 SDValue Callee,
4710 PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
4747 static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
4766 SDValue Arg;
4767 SDValue FrameIdxOp;
4777 SelectionDAG &DAG, SDValue Chain,
4779 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
4781 SDValue Arg = TailCallArgs[i].Arg;
4782 SDValue FIN = TailCallArgs[i].FrameIdxOp;
4793 static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
4793 static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
4794 SDValue OldRetAddr, SDValue OldFP,
4794 SDValue OldRetAddr, SDValue OldFP,
4807 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT);
4817 SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT);
4830 SDValue Arg, int SPDiff, unsigned ArgOffset,
4836 SDValue FIN = DAG.getFrameIndex(FI, VT);
4847 SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
4848 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
4848 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
4849 SDValue &FPOpOut, const SDLoc &dl) const {
4874 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
4874 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
4874 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
4875 SDValue Chain, ISD::ArgFlagsTy Flags,
4877 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
4886 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
4886 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
4887 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
4888 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
4893 SDValue StackPtr;
4909 PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain,
4909 PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain,
4910 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
4911 SDValue FPOp,
4915 SmallVector<SDValue, 8> MemOpChains2;
4934 static bool isFunctionGlobalAddress(SDValue Callee) {
4947 PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain,
4947 PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain,
4947 PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain,
4948 SDValue CallSeqStart, const SDLoc &dl, int SPDiff, bool isTailCall,
4950 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
4951 SmallVectorImpl<SDValue> &Ops, std::vector<EVT> &NodeTys,
5024 SDValue MTCTROps[] = {Chain, Callee, InFlag};
5056 SDValue LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-1);
5066 SDValue LoadFuncPtr = DAG.getLoad(MVT::i64, dl, LDChain, Callee, MPI,
5070 SDValue PtrOff = DAG.getIntPtrConstant(16, dl);
5071 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff);
5072 SDValue LoadEnvPtr =
5076 SDValue TOCOff = DAG.getIntPtrConstant(8, dl);
5077 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, TOCOff);
5078 SDValue TOCPtr =
5083 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, PPC::X2, TOCPtr,
5091 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr,
5153 SDValue PPCTargetLowering::LowerCallResult(
5154 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
5154 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
5156 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
5171 SDValue Val;
5174 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
5179 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
5217 SDValue PPCTargetLowering::FinishCall(
5220 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue InFlag,
5220 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue InFlag,
5221 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5221 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5221 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5223 SmallVectorImpl<SDValue> &InVals, ImmutableCallSite CS) const {
5225 SmallVector<SDValue, 8> Ops;
5294 SDValue StackPtr = DAG.getRegister(PPC::X1, PtrVT);
5296 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
5297 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, TOCOff);
5335 SDValue
5337 SmallVectorImpl<SDValue> &InVals) const {
5341 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
5343 SDValue Chain = CLI.Chain;
5344 SDValue Callee = CLI.Callee;
5411 SDValue PPCTargetLowering::LowerCall_32SVR4(
5412 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
5412 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
5415 const SmallVectorImpl<SDValue> &OutVals,
5417 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5507 SDValue CallSeqStart = Chain;
5511 SDValue LROp, FPOp;
5517 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
5519 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
5521 SmallVector<SDValue, 8> MemOpChains;
5532 SDValue Arg = OutVals[RealArgIdx];
5547 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
5553 SDValue MemcpyCall =
5559 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0,
5584 SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
5599 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
5618 SDValue InFlag;
5629 SDValue Ops[] = { Chain, InFlag };
5649 SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
5650 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
5650 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
5650 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
5652 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff,
5657 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0,
5664 SDValue PPCTargetLowering::LowerCall_64SVR4(
5665 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
5665 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
5668 const SmallVectorImpl<SDValue> &OutVals,
5670 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5852 SDValue CallSeqStart = Chain;
5856 SDValue LROp, FPOp;
5862 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
5870 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
5873 SmallVector<SDValue, 8> MemOpChains;
5875 SDValue Arg = OutVals[i];
5882 SDValue PtrOff;
5936 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
5947 SDValue AddPtr = PtrOff;
5949 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
5987 SDValue AddPtr = PtrOff;
5989 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType());
5997 SDValue Load =
6010 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
6011 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
6013 SDValue Load =
6086 SDValue ArgVal;
6100 SDValue Lo, Hi;
6130 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
6175 SDValue Store =
6179 SDValue Load =
6188 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
6190 SDValue Load =
6231 SDValue Store =
6235 SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store,
6244 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
6246 SDValue Load =
6292 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
6295 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
6296 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
6309 SDValue InFlag;
6325 SDValue PPCTargetLowering::LowerCall_Darwin(
6326 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
6326 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
6329 const SmallVectorImpl<SDValue> &OutVals,
6331 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
6412 SDValue CallSeqStart = Chain;
6416 SDValue LROp, FPOp;
6422 SDValue StackPtr;
6453 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6456 SmallVector<SDValue, 8> MemOpChains;
6458 SDValue Arg = OutVals[i];
6463 SDValue PtrOff;
6486 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
6493 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
6495 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
6514 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
6515 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
6517 SDValue Load =
6553 SDValue Store =
6559 SDValue Load =
6565 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
6567 SDValue Load =
6610 SDValue Store =
6614 SDValue Load =
6623 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
6625 SDValue Load =
6659 SDValue Arg = OutVals[i];
6664 SDValue PtrOff;
6690 SDValue InFlag;
6781 SDValue PPCTargetLowering::LowerCall_AIX(
6782 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
6782 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
6785 const SmallVectorImpl<SDValue> &OutVals,
6787 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
6832 SDValue CallSeqStart = Chain;
6834 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6837 SDValue Arg = OutVals[VA.getValNo()];
6860 SDValue InFlag;
6886 SDValue
6887 PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
6890 const SmallVectorImpl<SDValue> &OutVals,
6900 SDValue Flag;
6901 SmallVector<SDValue, 4> RetOps(1, Chain);
6908 SDValue Arg = OutVals[RealResIdx];
6926 SDValue SVal =
6970 SDValue
6971 PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
6979 SDValue Chain = Op.getOperand(0);
6980 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
6982 SDValue Ops[2] = {Chain, FPSIdx};
6987 SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
6987 SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
6998 SDValue StackPtr = DAG.getRegister(SP, PtrVT);
7001 SDValue Chain = Op.getOperand(0);
7002 SDValue SaveSP = Op.getOperand(1);
7005 SDValue LoadLinkSP =
7015 SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
7037 SDValue
7060 SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
7060 SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
7063 SDValue Chain = Op.getOperand(0);
7064 SDValue Size = Op.getOperand(1);
7070 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT,
7073 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7075 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
7080 SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
7080 SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
7091 SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
7091 SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
7099 SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
7099 SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
7106 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
7106 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
7118 SDValue Chain = LD->getChain();
7119 SDValue BasePtr = LD->getBasePtr();
7122 SDValue NewLD =
7125 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD);
7127 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
7131 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
7131 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
7143 SDValue Chain = ST->getChain();
7144 SDValue BasePtr = ST->getBasePtr();
7145 SDValue Value = ST->getValue();
7154 SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
7154 SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
7163 SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
7163 SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
7188 SDValue N1 = Op.getOperand(0);
7191 SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL);
7214 SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc);
7220 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
7220 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
7239 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
7240 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3);
7266 SDValue Sel1;
7302 SDValue Cmp;
7345 void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
7349 SDValue Src = Op.getOperand(0);
7353 SDValue Tmp;
7375 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64);
7381 SDValue Chain;
7386 SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr };
7408 SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
7408 SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
7412 SDValue Src = Op.getOperand(0);
7417 SDValue Tmp;
7440 SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
7440 SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
7452 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
7455 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
7460 SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi);
7468 SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128);
7472 SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128,
7477 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32,
7505 bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
7551 void PPCTargetLowering::spliceIntoChain(SDValue ResChain,
7552 SDValue NewResChain,
7559 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7571 bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
7601 SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
7601 SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
7609 SDValue FP;
7610 SDValue Src = Op.getOperand(0);
7630 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
7630 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
7641 SmallVector<SDValue, 16> Ops(NumConcat);
7643 SDValue UndefVec = DAG.getUNDEF(VecVT);
7650 SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
7650 SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
7662 SDValue Wide = widenVec(DAG, Op.getOperand(0), dl);
7680 SDValue ShuffleSrc2 =
7682 SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV);
7686 SDValue Extend;
7697 SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
7697 SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
7715 SDValue Value = Op.getOperand(0);
7721 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
7762 SDValue SINT = Op.getOperand(0);
7783 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64,
7799 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64,
7810 SDValue Bits;
7822 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
7832 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
7846 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
7848 SDValue Store =
7865 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
7873 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits);
7891 SDValue Ld;
7898 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
7900 SDValue Store =
7918 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
7930 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
7932 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64,
7936 SDValue Store = DAG.getStore(
7947 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld);
7954 SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
7954 SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
7985 SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None);
7989 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
7990 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain, StackSlot,
7994 SDValue Four = DAG.getConstant(4, dl, PtrVT);
7995 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four);
7996 SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo());
7999 SDValue CWD1 =
8002 SDValue CWD2 =
8010 SDValue RetVal =
8017 SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
8017 SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
8027 SDValue Lo = Op.getOperand(0);
8028 SDValue Hi = Op.getOperand(1);
8029 SDValue Amt = Op.getOperand(2);
8032 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
8034 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt);
8035 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1);
8036 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3);
8037 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
8039 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5);
8040 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
8041 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt);
8042 SDValue OutOps[] = { OutLo, OutHi };
8046 SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
8046 SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
8056 SDValue Lo = Op.getOperand(0);
8057 SDValue Hi = Op.getOperand(1);
8058 SDValue Amt = Op.getOperand(2);
8061 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
8063 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
8064 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
8065 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
8066 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
8068 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5);
8069 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
8070 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt);
8071 SDValue OutOps[] = { OutLo, OutHi };
8075 SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
8075 SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
8084 SDValue Lo = Op.getOperand(0);
8085 SDValue Hi = Op.getOperand(1);
8086 SDValue Amt = Op.getOperand(2);
8089 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
8091 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
8092 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
8093 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
8094 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
8096 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5);
8097 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt);
8098 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT),
8100 SDValue OutOps[] = { OutLo, OutHi };
8110 static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT,
8132 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
8132 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
8141 static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
8141 static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
8141 static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
8151 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
8151 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
8151 static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
8152 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl,
8161 static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
8161 static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
8161 static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
8170 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops);
8198 SDValue Op0 = V->getOperand(0);
8228 SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
8228 SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
8231 SDValue Op0 = Op->getOperand(0);
8244 static const SDValue *getNormalLoadInput(const SDValue &Op) {
8244 static const SDValue *getNormalLoadInput(const SDValue &Op) {
8245 const SDValue *InputLoad = &Op;
8261 SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
8261 SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
8276 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
8307 SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()),
8310 SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
8317 SmallVector<SDValue, 4> Stores;
8322 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
8331 SDValue StoreValue = BVN->getOperand(i);
8340 SDValue StoreChain;
8351 SDValue Ops[] = {StoreChain,
8356 SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN,
8362 SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64);
8379 const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0));
8394 SDValue Ops[] = {
8426 SDValue Z = DAG.getConstant(0, dl, MVT::v4i32);
8438 SmallVector<SDValue, 16> Ops(16, DAG.getConstant(SplatBits,
8440 SDValue NewBV = DAG.getBuildVector(MVT::v16i8, dl, Ops);
8476 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32);
8479 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32);
8480 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize);
8492 SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl);
8495 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV,
8520 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
8531 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
8542 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
8554 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
8565 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
8571 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
8577 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
8588 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8588 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8589 SDValue RHS, SelectionDAG &DAG,
8614 SDValue OpLHS, OpRHS;
8659 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs);
8666 SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N,
8671 SDValue V1 = N->getOperand(0);
8672 SDValue V2 = N->getOperand(1);
8755 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2,
8767 SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
8777 SDValue V1 = N->getOperand(0);
8778 SDValue V2 = N->getOperand(1);
8860 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
8863 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2,
8865 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl);
8866 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2,
8870 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2);
8871 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2,
8880 SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
8880 SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
8883 SDValue V1 = Op.getOperand(0);
8884 SDValue V2 = Op.getOperand(1);
8895 const SDValue *InputLoad = getNormalLoadInput(V1);
8911 SDValue BasePtr = LD->getBasePtr();
8915 SDValue Ops[] = {
8922 SDValue LdSplt =
8935 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
8936 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2);
8938 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2,
8940 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl,
8944 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2,
8950 SDValue NewISDNode;
8962 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
8963 SDValue Conv2 =
8966 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2,
8975 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1);
8976 SDValue Conv2 =
8979 SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2,
8986 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
8987 SDValue ReveHWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v8i16, Conv);
8990 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
8991 SDValue ReveWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v4i32, Conv);
8994 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1);
8995 SDValue ReveDWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v2i64, Conv);
8998 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1);
8999 SDValue ReveQWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v1i128, Conv);
9008 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
9009 SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv,
9016 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1);
9017 SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv);
9053 SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64,
9171 SmallVector<SDValue, 16> ResultMask;
9184 SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask);
9196 static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc,
9442 SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
9442 SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
9465 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(),
9472 SDValue Ops[] = {
9478 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops);
9482 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32,
9519 SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
9519 SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
9541 SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const {
9541 SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const {
9554 SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
9554 SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
9572 SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
9572 SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
9582 SDValue CmpOp = Op.getOperand(2);
9590 SDValue NewCmpOp =
9595 SmallVector<SDValue, 4> Ops;
9606 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
9606 SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
9613 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
9616 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx,
9622 SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
9622 SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
9634 SDValue V1 = Op.getOperand(0);
9635 SDValue V2 = Op.getOperand(1);
9638 SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2);
9651 SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
9651 SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
9659 SDValue Value = N->getOperand(0);
9671 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
9685 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
9687 SDValue StoreChain = DAG.getEntryNode();
9688 SDValue Ops[] = {StoreChain,
9698 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
9701 SDValue IntVal =
9711 SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
9711 SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
9715 SDValue LoadChain = LN->getChain();
9716 SDValue BasePtr = LN->getBasePtr();
9731 SDValue Vals[4], LoadChains[4];
9733 SDValue Load;
9761 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
9762 SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals);
9765 SDValue RetOps[] = { Value, Vals[0].getValue(1), TF };
9769 SDValue RetOps[] = { Value, TF };
9779 SDValue VectElmts[4], VectElmtChains[4];
9781 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
9792 SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts);
9794 SDValue RVals[] = { Value, LoadChain };
9799 SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
9799 SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
9803 SDValue StoreChain = SN->getChain();
9804 SDValue BasePtr = SN->getBasePtr();
9805 SDValue Value = SN->getValue();
9820 SDValue Stores[4];
9822 SDValue Ex = DAG.getNode(
9825 SDValue Store;
9851 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
9854 SDValue RetOps[] = { TF, Stores[0].getValue(1) };
9871 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
9885 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
9887 SDValue Ops[] = {StoreChain,
9896 SDValue Loads[4], LoadChains[4];
9899 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
9909 SDValue Stores[4];
9911 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
9925 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
9925 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
9928 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
9930 SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl);
9931 SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt.
9933 SDValue RHSSwap = // = vrlw RHS, 16
9943 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh,
9946 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm,
9953 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
9955 SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl);
9960 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
9964 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub,
9969 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub,
9996 SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
9996 SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
10016 SDValue X = Op.getOperand(0);
10017 SDValue Zero = DAG.getConstant(0, dl, VT);
10018 SDValue Y = DAG.getNode(ISD::SUB, dl, VT, Zero, X);
10035 SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
10035 SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
10046 SDValue Op0 = Op.getOperand(0);
10078 SDValue NewLoad[2];
10081 SDValue LdOp = Op0.getOperand(i);
10086 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
10091 SDValue NewOp =
10099 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
10100 SDValue NewLd = DAG.getMemIntrinsicNode(
10112 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10112 SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10187 SmallVectorImpl<SDValue>&Results,
10195 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0));
10212 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0),
10226 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG);
11645 SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
11645 SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
11667 SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
11667 SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
11709 static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
11709 static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
11721 static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
11727 SDValue BaseLoc = Base->getBasePtr();
11740 SDValue Base1 = Loc, Base2 = BaseLoc;
11766 SDValue Loc = LS->getBasePtr();
11875 SDValue Chain = LD->getChain();
11941 static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement,
11947 auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0),
11949 auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1),
11957 auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1);
11961 auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode,
11963 auto Final = Shifted;
11973 SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N,
12015 SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
12100 SmallVector<SDValue, 4> Inputs;
12101 SmallVector<SDValue, 8> BinOps, PromOps;
12121 SDValue BinOp = BinOps.back();
12225 for (auto &PromOp : PromOps)
12234 SDValue PromOp = PromOpHandles.back().getValue();
12248 SDValue RepValue = PromOp.getOperand(0);
12275 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(),
12296 SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
12330 SmallVector<SDValue, 4> Inputs;
12331 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps;
12337 SDValue BinOp = BinOps.back();
12464 SDValue InSrc = Inputs[i].getOperand(0);
12479 for (auto &PromOp : PromOps)
12487 SDValue PromOp = PromOpHandles.back().getValue();
12522 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(),
12571 SDValue ShiftCst =
12579 SDValue PPCTargetLowering::combineSetCC(SDNode *N,
12586 SDValue LHS = N->getOperand(0);
12587 SDValue RHS = N->getOperand(1);
12602 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1));
12611 static bool isFPExtLoad(SDValue Op) {
12625 SDValue PPCTargetLowering::
12634 SDValue FirstInput = N->getOperand(0);
12649 SmallVector<SDValue, 4> Ops;
12652 SDValue NextOp = N->getOperand(i);
12676 SDValue In = N->getOperand(i).getOperand(0);
12683 SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl,
12700 SDValue BV = DAG.getBuildVector(NewVT, dl, Ops);
12712 static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
12725 SDValue FirstInput = N->getOperand(0);
12743 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) :
12748 SDValue PreviousInput =
12770 SDValue FirstLoadOp =
12772 SDValue LastLoadOp =
12786 SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(),
12802 static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG,
12803 SDValue Input, uint64_t Elems,
12822 SDValue Shuffle =
12827 SDValue BV = DAG.getNode(PPCISD::SExtVElems, dl, Ty, Shuffle);
12836 static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
12857 SDValue Input;
12868 SDValue Extract = Op.getOperand(0);
12930 SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N,
12944 SDValue FirstInput = N->getOperand(0);
12946 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI);
12953 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG);
12982 SDValue Ext1 = FirstInput.getOperand(0);
12983 SDValue Ext2 = N->getOperand(1).getOperand(0);
13006 SDValue SrcVec = Ext1.getOperand(0);
13013 SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
13024 SDValue Op(N, 0);
13034 SDValue FirstOperand(Op.getOperand(0));
13044 SDValue WidthConst =
13048 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst };
13049 SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl,
13055 SDValue ExtOps[] = { Ld, WidthConst };
13056 SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps);
13089 SDValue Src = Op.getOperand(0).getOperand(0);
13102 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src);
13103 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp);
13119 SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
13123 SDValue Chain;
13124 SDValue Base;
13162 SDValue LoadOps[] = { Chain, Base };
13163 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl,
13169 SDValue Swap = DAG.getNode(
13175 SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap);
13187 SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
13191 SDValue Chain;
13192 SDValue Base;
13223 SDValue Src = N->getOperand(SrcOpnd);
13239 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl,
13243 SDValue StoreOps[] = { Chain, Swap, Base };
13244 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl,
13252 SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
13262 SDValue Val = N->getOperand(1).getOperand(0);
13296 SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2),
13309 SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN,
13346 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()};
13354 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(0),
13364 SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
13402 if (SDValue CSCC = combineSetCC(N, DCI))
13422 SDValue Val= combineStoreFPToInt(N, DCI);
13429 SDValue Val= combineVReverseMemOP(SVN, cast<LSBaseSDNode>(N), DCI);
13446 SDValue BSwapOp = N->getOperand(1).getOperand(0);
13462 SDValue Ops[] = {
13479 SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64);
13581 SDValue BasePtr = LD->getBasePtr();
13592 SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr,
13595 SDValue AddPtr =
13598 SDValue FloatLoad2 = DAG.getLoad(
13634 SDValue Chain = LD->getChain();
13635 SDValue Ptr = LD->getBasePtr();
13683 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy);
13697 SDValue LDXIntID =
13699 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
13700 SDValue BaseLoad =
13722 SDValue Increment =
13729 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
13730 SDValue ExtraLoad =
13735 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
13743 SDValue Perm;
13775 SDValue Add = N->getOperand(1);
13826 SDValue V1 = N->getOperand(1);
13827 SDValue V2 = N->getOperand(2);
13888 SDValue Load = N->getOperand(0);
13891 SDValue Ops[] = {
13896 SDValue BSLoad =
13903 SDValue ResVal = BSLoad;
13970 SDValue Cond = N->getOperand(1);
13971 SDValue Target = N->getOperand(2);
13994 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3);
14049 SDValue Ops[] = {
14055 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops);
14093 SDValue
14106 SDValue N0 = N->getOperand(0);
14110 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT);
14112 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt);
14127 void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
14392 void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
14394 std::vector<SDValue>&Ops,
14396 SDValue Result;
14505 SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
14505 SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
14525 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
14526 SDValue Offset =
14535 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
14540 SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
14540 SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
14560 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg,
14592 bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
14873 bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
15100 static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N,
15102 SDValue N0 = N->getOperand(0);
15103 SDValue N1 = N->getOperand(1);
15132 SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const {
15133 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG))
15136 SDValue N0 = N->getOperand(0);
15146 SDValue ExtsSrc = N0.getOperand(0);
15152 SDValue ShiftBy = SDValue(CN1, 0);
15162 SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const {
15163 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG))
15169 SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const {
15170 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG))
15180 static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
15185 SDValue LHS = N->getOperand(0);
15186 SDValue RHS = N->getOperand(1);
15193 SDValue Cmp = Op.getOperand(0);
15219 SDValue Cmp = RHS.getOperand(0);
15220 SDValue Z = Cmp.getOperand(0);
15235 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z,
15237 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
15238 SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue),
15250 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z,
15252 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
15253 SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue),
15263 SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const {
15264 if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget))
15279 SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N,
15284 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI))
15289 SDValue Op0 = N->getOperand(0);
15314 SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0));
15322 SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const {
15373 SDValue Op0 = N->getOperand(0);
15374 SDValue Op1 =
15377 SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1);
15390 SDValue Op0 = N->getOperand(0);
15391 SDValue Op1 =
15471 SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const {
15514 SDValue PPCTargetLowering::combineVSelect(SDNode *N,
15522 SDValue Cond = N->getOperand(0);
15523 SDValue TrueOpnd = N->getOperand(1);
15524 SDValue FalseOpnd = N->getOperand(2);
15554 SDValue CmpOpnd1 = Cond.getOperand(0);
15555 SDValue CmpOpnd2 = Cond.getOperand(1);
lib/Target/PowerPC/PPCISelLowering.h 594 SDValue get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG);
653 bool hasAndNotCompare(SDValue) const override {
687 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base,
688 SDValue &Offset,
694 bool SelectAddressEVXRegReg(SDValue N, SDValue &Base, SDValue &Index,
694 bool SelectAddressEVXRegReg(SDValue N, SDValue &Base, SDValue &Index,
694 bool SelectAddressEVXRegReg(SDValue N, SDValue &Base, SDValue &Index,
701 bool SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index,
701 bool SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index,
701 bool SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index,
710 bool SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base,
710 bool SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base,
710 bool SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base,
716 bool SelectAddressRegRegOnly(SDValue N, SDValue &Base, SDValue &Index,
716 bool SelectAddressRegRegOnly(SDValue N, SDValue &Base, SDValue &Index,
716 bool SelectAddressRegRegOnly(SDValue N, SDValue &Base, SDValue &Index,
723 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
723 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
728 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
731 SDValue expandVSXLoadForLE(SDNode *N, DAGCombinerInfo &DCI) const;
732 SDValue expandVSXStoreForLE(SDNode *N, DAGCombinerInfo &DCI) const;
734 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
736 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
742 void computeKnownBitsForTargetNode(const SDValue Op,
800 void LowerAsmOperandForConstraint(SDValue Op,
802 std::vector<SDValue> &Ops,
844 bool isZExtFree(SDValue Val, EVT VT2) const override;
870 bool isAccessedAsGotIndirect(SDValue N) const;
953 SDValue getPICJumpTableRelocBase(SDValue Table,
953 SDValue getPICJumpTableRelocBase(SDValue Table,
961 SDValue Ptr;
962 SDValue Chain;
963 SDValue ResChain;
988 bool canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI,
991 void spliceIntoChain(SDValue ResChain, SDValue NewResChain,
991 void spliceIntoChain(SDValue ResChain, SDValue NewResChain,
994 void LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
996 SDValue LowerFP_TO_INTDirectMove(SDValue Op, SelectionDAG &DAG,
996 SDValue LowerFP_TO_INTDirectMove(SDValue Op, SelectionDAG &DAG,
999 bool directMoveIsProfitable(const SDValue &Op) const;
1000 SDValue LowerINT_TO_FPDirectMove(SDValue Op, SelectionDAG &DAG,
1000 SDValue LowerINT_TO_FPDirectMove(SDValue Op, SelectionDAG &DAG,
1003 SDValue LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
1003 SDValue LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
1006 SDValue LowerTRUNCATEVector(SDValue Op, SelectionDAG &DAG) const;
1006 SDValue LowerTRUNCATEVector(SDValue Op, SelectionDAG &DAG) const;
1008 SDValue getFramePointerFrameIndex(SelectionDAG & DAG) const;
1009 SDValue getReturnAddrFrameIndex(SelectionDAG & DAG) const;
1012 IsEligibleForTailCallOptimization(SDValue Callee,
1020 SDValue Callee,
1028 SDValue EmitTailCallLoadFPAndRetAddr(SelectionDAG &DAG, int SPDiff,
1029 SDValue Chain, SDValue &LROpOut,
1029 SDValue Chain, SDValue &LROpOut,
1030 SDValue &FPOpOut,
1033 SDValue getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, SDValue GA) const;
1033 SDValue getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, SDValue GA) const;
1035 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1035 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1036 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1036 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1037 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1037 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1038 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1038 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1039 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1039 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1040 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1040 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1041 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1041 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1042 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1042 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1043 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1043 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1044 SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1044 SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1045 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1045 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1046 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1046 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1047 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
1047 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
1048 SDValue LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
1048 SDValue LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
1049 SDValue LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1049 SDValue LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1050 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1050 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1051 SDValue LowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
1051 SDValue LowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
1052 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
1052 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
1053 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
1053 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
1054 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1054 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1055 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
1055 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
1056 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
1056 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
1058 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1058 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1059 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
1059 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
1060 SDValue LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const;
1060 SDValue LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const;
1061 SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const;
1061 SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const;
1062 SDValue LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const;
1062 SDValue LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const;
1063 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1063 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1064 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
1064 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
1065 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1065 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1066 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1066 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1067 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1067 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1068 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
1068 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
1069 SDValue LowerREM(SDValue Op, SelectionDAG &DAG) const;
1069 SDValue LowerREM(SDValue Op, SelectionDAG &DAG) const;
1070 SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const;
1070 SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const;
1071 SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
1071 SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
1072 SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1072 SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1073 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
1073 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
1074 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
1074 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
1075 SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;
1075 SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;
1076 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1076 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1078 SDValue LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const;
1078 SDValue LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const;
1079 SDValue LowerVectorStore(SDValue Op, SelectionDAG &DAG) const;
1079 SDValue LowerVectorStore(SDValue Op, SelectionDAG &DAG) const;
1081 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1081 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1081 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1085 SmallVectorImpl<SDValue> &InVals) const;
1086 SDValue FinishCall(CallingConv::ID CallConv, const SDLoc &dl,
1089 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
1090 SDValue InFlag, SDValue Chain, SDValue CallSeqStart,
1090 SDValue InFlag, SDValue Chain, SDValue CallSeqStart,
1090 SDValue InFlag, SDValue Chain, SDValue CallSeqStart,
1091 SDValue &Callee, int SPDiff, unsigned NumBytes,
1093 SmallVectorImpl<SDValue> &InVals,
1096 SDValue
1097 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1100 SmallVectorImpl<SDValue> &InVals) const override;
1102 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
1103 SmallVectorImpl<SDValue> &InVals) const override;
1110 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1110 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1112 const SmallVectorImpl<SDValue> &OutVals,
1115 SDValue extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT,
1116 SelectionDAG &DAG, SDValue ArgVal,
1119 SDValue LowerFormalArguments_Darwin(
1120 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1122 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const;
1123 SDValue LowerFormalArguments_64SVR4(
1124 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1126 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const;
1127 SDValue LowerFormalArguments_32SVR4(
1128 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1130 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const;
1132 SDValue createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff,
1132 SDValue createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff,
1132 SDValue createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff,
1133 SDValue CallSeqStart,
1137 SDValue LowerCall_Darwin(SDValue Chain, SDValue Callee,
1137 SDValue LowerCall_Darwin(SDValue Chain, SDValue Callee,
1137 SDValue LowerCall_Darwin(SDValue Chain, SDValue Callee,
1141 const SmallVectorImpl<SDValue> &OutVals,
1144 SmallVectorImpl<SDValue> &InVals,
1146 SDValue LowerCall_64SVR4(SDValue Chain, SDValue Callee,
1146 SDValue LowerCall_64SVR4(SDValue Chain, SDValue Callee,
1146 SDValue LowerCall_64SVR4(SDValue Chain, SDValue Callee,
1150 const SmallVectorImpl<SDValue> &OutVals,
1153 SmallVectorImpl<SDValue> &InVals,
1155 SDValue LowerCall_32SVR4(SDValue Chain, SDValue Callee,
1155 SDValue LowerCall_32SVR4(SDValue Chain, SDValue Callee,
1155 SDValue LowerCall_32SVR4(SDValue Chain, SDValue Callee,
1159 const SmallVectorImpl<SDValue> &OutVals,
1162 SmallVectorImpl<SDValue> &InVals,
1164 SDValue LowerCall_AIX(SDValue Chain, SDValue Callee,
1164 SDValue LowerCall_AIX(SDValue Chain, SDValue Callee,
1164 SDValue LowerCall_AIX(SDValue Chain, SDValue Callee,
1168 const SmallVectorImpl<SDValue> &OutVals,
1171 SmallVectorImpl<SDValue> &InVals,
1174 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1174 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1175 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1175 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1176 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
1176 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
1178 SDValue DAGCombineExtBoolTrunc(SDNode *N, DAGCombinerInfo &DCI) const;
1179 SDValue DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const;
1180 SDValue DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const;
1181 SDValue combineStoreFPToInt(SDNode *N, DAGCombinerInfo &DCI) const;
1182 SDValue combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const;
1183 SDValue combineSHL(SDNode *N, DAGCombinerInfo &DCI) const;
1184 SDValue combineSRA(SDNode *N, DAGCombinerInfo &DCI) const;
1185 SDValue combineSRL(SDNode *N, DAGCombinerInfo &DCI) const;
1186 SDValue combineMUL(SDNode *N, DAGCombinerInfo &DCI) const;
1187 SDValue combineADD(SDNode *N, DAGCombinerInfo &DCI) const;
1188 SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const;
1189 SDValue combineSetCC(SDNode *N, DAGCombinerInfo &DCI) const;
1190 SDValue combineABS(SDNode *N, DAGCombinerInfo &DCI) const;
1191 SDValue combineVSelect(SDNode *N, DAGCombinerInfo &DCI) const;
1192 SDValue combineVReverseMemOP(ShuffleVectorSDNode *SVN, LSBaseSDNode *LSBase,
1198 SDValue ConvertSETCCToSubtract(SDNode *N, DAGCombinerInfo &DCI) const;
1200 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1200 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1203 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1203 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1207 SDValue
1215 SDValue lowerToVINSERTH(ShuffleVectorSDNode *N, SelectionDAG &DAG) const;
1220 SDValue lowerToVINSERTB(ShuffleVectorSDNode *N, SelectionDAG &DAG) const;
1238 bool isIntS16Immediate(SDValue Op, int16_t &Imm);
lib/Target/RISCV/RISCVISelDAGToDAG.cpp 49 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
50 std::vector<SDValue> &OutOps) override;
52 bool SelectAddrFI(SDValue Addr, SDValue &Base);
52 bool SelectAddrFI(SDValue Addr, SDValue &Base);
72 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
74 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
117 SDValue New = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), SDLoc(Node),
130 SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
132 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
139 SDValue Op0 = Node->getOperand(0);
140 SDValue Op1 = Node->getOperand(1);
151 SDValue ShAmtVal =
174 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
174 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
192 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
192 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
249 SDValue Base = N->getOperand(BaseOpIdx);
255 SDValue ImmOperand = Base.getOperand(1);
lib/Target/RISCV/RISCVISelLowering.cpp 306 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
332 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
332 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
368 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
368 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
399 SDValue Op0 = Op.getOperand(0);
402 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
403 SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
409 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
414 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
420 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
427 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
433 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
452 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
453 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
454 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
461 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
467 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
467 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
477 SDValue Addr = getAddr(N, DAG, IsLocal);
489 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
489 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
496 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
496 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
503 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
516 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
517 SDValue Load =
521 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
529 SDValue AddrHi =
531 SDValue AddrAdd =
533 SDValue AddrLo =
536 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
537 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
538 SDValue MNAdd = SDValue(
544 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
554 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
555 SDValue Load =
576 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
576 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
589 SDValue Addr;
613 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
613 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
614 SDValue CondV = Op.getOperand(0);
615 SDValue TrueV = Op.getOperand(1);
616 SDValue FalseV = Op.getOperand(2);
628 SDValue LHS = CondV.getOperand(0);
629 SDValue RHS = CondV.getOperand(1);
635 SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
637 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
644 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
645 SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
648 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
653 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
653 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
658 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
668 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
668 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
679 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
683 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
691 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
691 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
708 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
709 SDValue Offset = DAG.getConstant(Off, DL, VT);
721 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
721 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
724 SDValue Lo = Op.getOperand(0);
725 SDValue Hi = Op.getOperand(1);
726 SDValue Shamt = Op.getOperand(2);
736 SDValue Zero = DAG.getConstant(0, DL, VT);
737 SDValue One = DAG.getConstant(1, DL, VT);
738 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
739 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
740 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
741 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
743 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
744 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
745 SDValue ShiftRightLo =
747 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
748 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
749 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
751 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
756 SDValue Parts[2] = {Lo, Hi};
760 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
760 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
763 SDValue Lo = Op.getOperand(0);
764 SDValue Hi = Op.getOperand(1);
765 SDValue Shamt = Op.getOperand(2);
786 SDValue Zero = DAG.getConstant(0, DL, VT);
787 SDValue One = DAG.getConstant(1, DL, VT);
788 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
789 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
790 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
791 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
793 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
794 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
795 SDValue ShiftLeftHi =
797 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
798 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
799 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
800 SDValue HiFalse =
803 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
808 SDValue Parts[2] = {Lo, Hi};
838 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG) {
841 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
842 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
843 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
850 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
852 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
853 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
854 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
855 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
861 SmallVectorImpl<SDValue> &Results,
872 SDValue RCW =
912 SDValue Op0 = N->getOperand(0);
915 SDValue FPConv =
923 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
931 SDValue Op0 = N->getOperand(0);
944 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
945 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
956 SDValue NewSplitF64 =
959 SDValue Lo = NewSplitF64.getValue(0);
960 SDValue Hi = NewSplitF64.getValue(1);
963 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
968 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
976 SDValue LHS = N->getOperand(0);
977 SDValue RHS = N->getOperand(1);
987 SDValue Op0 = N->getOperand(0);
992 SDValue AExtOp =
1004 SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64,
1029 SDValue N0 = N->getOperand(0);
1069 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1667 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
1667 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
1687 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
1687 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
1692 SDValue Val;
1720 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
1720 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
1742 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
1742 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
1751 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1752 SDValue Val;
1770 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
1770 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
1781 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1790 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
1791 SDValue Hi;
1795 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1865 SDValue RISCVTargetLowering::LowerFormalArguments(
1866 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1868 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1898 std::vector<SDValue> OutChains;
1911 SDValue ArgValue;
1932 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1985 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
1987 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1988 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
2015 auto &Callee = CLI.Callee;
2087 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
2088 SmallVectorImpl<SDValue> &InVals) const {
2092 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2094 SDValue Chain = CLI.Chain;
2095 SDValue Callee = CLI.Callee;
2127 SmallVector<SDValue, 8> ByValArgs;
2133 SDValue Arg = OutVals[i];
2138 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2139 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
2153 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
2154 SmallVector<SDValue, 8> MemOpChains;
2155 SDValue StackPtr;
2158 SDValue ArgValue = OutVals[i];
2165 SDValue SplitF64 = DAG.getNode(
2167 SDValue Lo = SplitF64.getValue(0);
2168 SDValue Hi = SplitF64.getValue(1);
2197 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
2207 SDValue PartValue = OutVals[i + 1];
2209 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2236 SDValue Address =
2250 SDValue Glue;
2290 SmallVector<SDValue, 8> Ops;
2337 SDValue RetValue =
2345 SDValue RetValue2 =
2377 SDValue
2378 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2381 const SmallVectorImpl<SDValue> &OutVals,
2396 SDValue Glue;
2397 SmallVector<SDValue, 4> RetOps(1, Chain);
2401 SDValue Val = OutVals[i];
2408 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
2410 SDValue Lo = SplitF64.getValue(0);
2411 SDValue Hi = SplitF64.getValue(1);
2477 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
2684 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2684 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
lib/Target/RISCV/RISCVISelLowering.h 75 bool isZExtFree(SDValue Val, EVT VT2) const override;
81 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
81 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
82 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
85 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
87 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
103 void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
104 std::vector<SDValue> &Ops,
158 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
158 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
162 SmallVectorImpl<SDValue> &InVals) const override;
167 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
167 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
169 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
171 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
172 SmallVectorImpl<SDValue> &InVals) const override;
179 SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const;
181 SDValue getStaticTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG,
183 SDValue getDynamicTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG) const;
186 SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
186 SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
187 SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
187 SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
188 SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
188 SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
189 SDValue lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
189 SDValue lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
190 SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
190 SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
191 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
191 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
192 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
192 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
193 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
193 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
194 SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
194 SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
195 SDValue lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, bool IsSRA) const;
195 SDValue lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, bool IsSRA) const;
217 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
lib/Target/Sparc/SparcISelDAGToDAG.cpp 46 bool SelectADDRrr(SDValue N, SDValue &R1, SDValue &R2);
46 bool SelectADDRrr(SDValue N, SDValue &R1, SDValue &R2);
46 bool SelectADDRrr(SDValue N, SDValue &R1, SDValue &R2);
47 bool SelectADDRri(SDValue N, SDValue &Base, SDValue &Offset);
47 bool SelectADDRri(SDValue N, SDValue &Base, SDValue &Offset);
47 bool SelectADDRri(SDValue N, SDValue &Base, SDValue &Offset);
51 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
53 std::vector<SDValue> &OutOps) override;
75 bool SparcDAGToDAGISel::SelectADDRri(SDValue Addr,
76 SDValue &Base, SDValue &Offset) {
76 SDValue &Base, SDValue &Offset) {
120 bool SparcDAGToDAGISel::SelectADDRrr(SDValue Addr, SDValue &R1, SDValue &R2) {
120 bool SparcDAGToDAGISel::SelectADDRrr(SDValue Addr, SDValue &R1, SDValue &R2) {
120 bool SparcDAGToDAGISel::SelectADDRrr(SDValue Addr, SDValue &R1, SDValue &R2) {
157 std::vector<SDValue> AsmNodeOperands;
171 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
177 SDValue op = N->getOperand(i);
195 SDValue op = N->getOperand(++i);
222 SDValue V0 = N->getOperand(i+1);
223 SDValue V1 = N->getOperand(i+2);
226 SDValue PairedReg;
236 SDValue Chain = SDValue(N,0);
239 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::v2i32,
243 SDValue Sub0 = CurDAG->getTargetExtractSubreg(SP::sub_even, dl, MVT::i32,
245 SDValue Sub1 = CurDAG->getTargetExtractSubreg(SP::sub_odd, dl, MVT::i32,
247 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
249 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
252 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
259 SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
262 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
264 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
266 SDValue Pair = SDValue(
315 SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
347 SDValue DivLHS = N->getOperand(0);
348 SDValue DivRHS = N->getOperand(1);
351 SDValue TopPart;
377 SparcDAGToDAGISel::SelectInlineAsmMemoryOperand(const SDValue &Op,
379 std::vector<SDValue> &OutOps) {
380 SDValue Op0, Op1;
lib/Target/Sparc/SparcISelLowering.cpp 196 SDValue
197 SparcTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
200 const SmallVectorImpl<SDValue> &OutVals,
207 SDValue
208 SparcTargetLowering::LowerReturn_32(SDValue Chain, CallingConv::ID CallConv,
211 const SmallVectorImpl<SDValue> &OutVals,
225 SDValue Flag;
226 SmallVector<SDValue, 4> RetOps(1, Chain);
237 SDValue Arg = OutVals[realRVLocIdx];
244 SDValue Part0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
247 SDValue Part1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
273 SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, PtrVT);
292 SDValue
293 SparcTargetLowering::LowerReturn_64(SDValue Chain, CallingConv::ID CallConv,
296 const SmallVectorImpl<SDValue> &OutVals,
308 SDValue Flag;
309 SmallVector<SDValue, 4> RetOps(1, Chain);
319 SDValue OutVal = OutVals[i];
346 SDValue NV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, OutVals[i+1]);
369 SDValue SparcTargetLowering::LowerFormalArguments(
370 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
372 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
383 SDValue SparcTargetLowering::LowerFormalArguments_32(
384 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
386 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
409 SDValue FIPtr = DAG.getFrameIndex(FrameIdx, MVT::i32);
410 SDValue Arg =
422 SDValue HiVal = DAG.getCopyFromReg(Chain, dl, VRegHi, MVT::i32);
427 SDValue LoVal;
431 SDValue FIPtr = DAG.getFrameIndex(FrameIdx, MVT::i32);
442 SDValue WholeValue =
450 SDValue Arg = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
474 SDValue FIPtr = DAG.getFrameIndex(FI, PtrVT);
475 SDValue Load =
484 SDValue FIPtr = DAG.getFrameIndex(FI, PtrVT);
485 SDValue HiVal =
490 SDValue FIPtr2 = DAG.getFrameIndex(FI2, PtrVT);
492 SDValue LoVal =
498 SDValue WholeValue =
508 SDValue FIPtr = DAG.getFrameIndex(FI, PtrVT);
509 SDValue Load ;
530 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]);
552 std::vector<SDValue> OutChains;
557 SDValue Arg = DAG.getCopyFromReg(DAG.getRoot(), dl, VReg, MVT::i32);
561 SDValue FIPtr = DAG.getFrameIndex(FrameIdx, MVT::i32);
578 SDValue SparcTargetLowering::LowerFormalArguments_64(
579 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
581 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
602 SDValue Arg = DAG.getCopyFromReg(Chain, DL, VReg, VA.getLocVT());
669 SmallVector<SDValue, 8> OutChains;
672 SDValue VArg = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
686 SDValue
688 SmallVectorImpl<SDValue> &InVals) const {
694 static bool hasReturnsTwiceAttr(SelectionDAG &DAG, SDValue Callee,
716 SDValue
718 SmallVectorImpl<SDValue> &InVals) const {
722 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
724 SDValue Chain = CLI.Chain;
725 SDValue Callee = CLI.Callee;
748 SmallVector<SDValue, 8> ByValArgs;
754 SDValue Arg = OutVals[i];
760 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
761 SDValue SizeNode = DAG.getConstant(Size, dl, MVT::i32);
771 SDValue nullVal;
778 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
779 SmallVector<SDValue, 8> MemOpChains;
789 SDValue Arg = OutVals[realArgIdx];
822 SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
823 SDValue PtrOff = DAG.getIntPtrConstant(64, dl);
843 SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
844 SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
861 SDValue Part0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
864 SDValue Part1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
877 SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
878 SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
886 SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
887 SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
915 SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
916 SDValue PtrOff = DAG.getIntPtrConstant(VA.getLocMemOffset() + StackOffset,
932 SDValue InFlag;
952 SmallVector<SDValue, 8> Ops;
990 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2i32);
991 SDValue Lo = DAG.getCopyFromReg(
997 SDValue Hi = DAG.getCopyFromReg(
1092 SDValue
1094 SmallVectorImpl<SDValue> &InVals) const {
1097 SDValue Chain = CLI.Chain;
1130 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
1135 SmallVector<SDValue, 8> MemOpChains;
1139 SDValue Arg = CLI.OutVals[i];
1171 SDValue StackPtr = DAG.getRegister(SP::O6, PtrVT);
1172 SDValue HiPtrOff = DAG.getIntPtrConstant(StackOffset, DL);
1174 SDValue LoPtrOff = DAG.getIntPtrConstant(StackOffset + 8, DL);
1178 SDValue Store =
1181 SDValue Hi64 =
1183 SDValue Lo64 =
1202 SDValue NV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64,
1216 SDValue StackPtr = DAG.getRegister(SP::O6, PtrVT);
1219 SDValue PtrOff = DAG.getIntPtrConstant(VA.getLocMemOffset() +
1235 SDValue InGlue;
1245 SDValue Callee = CLI.Callee;
1254 SmallVector<SDValue, 8> Ops;
1308 SDValue RV;
1856 (const SDValue Op,
1881 static void LookThroughSetCC(SDValue &LHS, SDValue &RHS,
1881 static void LookThroughSetCC(SDValue &LHS, SDValue &RHS,
1892 SDValue CMPCC = LHS.getOperand(3);
1900 SDValue SparcTargetLowering::withTargetFlags(SDValue Op, unsigned TF,
1900 SDValue SparcTargetLowering::withTargetFlags(SDValue Op, unsigned TF,
1929 SDValue SparcTargetLowering::makeHiLoPair(SDValue Op,
1929 SDValue SparcTargetLowering::makeHiLoPair(SDValue Op,
1934 SDValue Hi = DAG.getNode(SPISD::Hi, DL, VT, withTargetFlags(Op, HiTF, DAG));
1935 SDValue Lo = DAG.getNode(SPISD::Lo, DL, VT, withTargetFlags(Op, LoTF, DAG));
1941 SDValue SparcTargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
1941 SDValue SparcTargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
1949 SDValue Idx;
1961 SDValue GlobalBase = DAG.getNode(SPISD::GLOBAL_BASE_REG, DL, VT);
1962 SDValue AbsAddr = DAG.getNode(ISD::ADD, DL, VT, GlobalBase, Idx);
1981 SDValue H44 = makeHiLoPair(Op, SparcMCExpr::VK_Sparc_H44,
1984 SDValue L44 = withTargetFlags(Op, SparcMCExpr::VK_Sparc_L44, DAG);
1990 SDValue Hi = makeHiLoPair(Op, SparcMCExpr::VK_Sparc_HH,
1993 SDValue Lo = makeHiLoPair(Op, SparcMCExpr::VK_Sparc_HI,
2000 SDValue SparcTargetLowering::LowerGlobalAddress(SDValue Op,
2000 SDValue SparcTargetLowering::LowerGlobalAddress(SDValue Op,
2005 SDValue SparcTargetLowering::LowerConstantPool(SDValue Op,
2005 SDValue SparcTargetLowering::LowerConstantPool(SDValue Op,
2010 SDValue SparcTargetLowering::LowerBlockAddress(SDValue Op,
2010 SDValue SparcTargetLowering::LowerBlockAddress(SDValue Op,
2015 SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op,
2015 SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op,
2042 SDValue HiLo = makeHiLoPair(Op, HiTF, LoTF, DAG);
2043 SDValue Base = DAG.getNode(SPISD::GLOBAL_BASE_REG, DL, PtrVT);
2044 SDValue Argument = DAG.getNode(SPISD::TLS_ADD, DL, PtrVT, Base, HiLo,
2047 SDValue Chain = DAG.getEntryNode();
2048 SDValue InFlag;
2053 SDValue Callee = DAG.getTargetExternalSymbol("__tls_get_addr", PtrVT);
2054 SDValue Symbol = withTargetFlags(Op, callTF, DAG);
2060 SDValue Ops[] = {Chain,
2071 SDValue Ret = DAG.getCopyFromReg(Chain, DL, SP::O0, PtrVT, InFlag);
2076 SDValue Hi = DAG.getNode(SPISD::Hi, DL, PtrVT,
2078 SDValue Lo = DAG.getNode(SPISD::Lo, DL, PtrVT,
2089 SDValue Base = DAG.getNode(SPISD::GLOBAL_BASE_REG, DL, PtrVT);
2096 SDValue TGA = makeHiLoPair(Op,
2099 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Base, TGA);
2100 SDValue Offset = DAG.getNode(SPISD::TLS_LD,
2110 SDValue Hi = DAG.getNode(SPISD::Hi, DL, PtrVT,
2112 SDValue Lo = DAG.getNode(SPISD::Lo, DL, PtrVT,
2114 SDValue Offset = DAG.getNode(ISD::XOR, DL, PtrVT, Hi, Lo);
2120 SDValue SparcTargetLowering::LowerF128_LibCallArg(SDValue Chain,
2120 SDValue SparcTargetLowering::LowerF128_LibCallArg(SDValue Chain,
2121 ArgListTy &Args, SDValue Arg,
2135 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2146 SDValue
2147 SparcTargetLowering::LowerF128Op(SDValue Op, SelectionDAG &DAG,
2156 SDValue Callee = DAG.getExternalSymbol(LibFuncName, PtrVT);
2159 SDValue Chain = DAG.getEntryNode();
2160 SDValue RetPtr;
2184 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
2184 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
2199 SDValue SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
2199 SDValue SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
2199 SDValue SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
2224 SDValue Callee = DAG.getExternalSymbol(LibCall, PtrVT);
2227 SDValue Chain = DAG.getEntryNode();
2235 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
2235 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
2238 SDValue Result = CallInfo.first;
2242 SDValue RHS = DAG.getTargetConstant(0, DL, Result.getValueType());
2247 SDValue Mask = DAG.getConstant(1, DL, Result.getValueType());
2249 SDValue RHS = DAG.getTargetConstant(0, DL, Result.getValueType());
2254 SDValue RHS = DAG.getTargetConstant(2, DL, Result.getValueType());
2259 SDValue RHS = DAG.getTargetConstant(1, DL, Result.getValueType());
2264 SDValue RHS = DAG.getTargetConstant(1, DL, Result.getValueType());
2270 SDValue RHS = DAG.getTargetConstant(3, DL, Result.getValueType());
2275 SDValue RHS = DAG.getTargetConstant(3, DL, Result.getValueType());
2280 SDValue Mask = DAG.getConstant(3, DL, Result.getValueType());
2282 SDValue RHS = DAG.getTargetConstant(0, DL, Result.getValueType());
2287 SDValue Mask = DAG.getConstant(3, DL, Result.getValueType());
2289 SDValue RHS = DAG.getTargetConstant(0, DL, Result.getValueType());
2296 static SDValue
2297 LowerF128_FPEXTEND(SDValue Op, SelectionDAG &DAG,
2312 static SDValue
2313 LowerF128_FPROUND(SDValue Op, SelectionDAG &DAG,
2330 static SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG,
2330 static SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG,
2359 static SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG,
2359 static SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG,
2382 SDValue Tmp = DAG.getNode(ISD::BITCAST, dl, floatVT, Op.getOperand(0));
2387 static SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG,
2387 static SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG,
2408 static SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG,
2408 static SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG,
2427 static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
2427 static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
2430 SDValue Chain = Op.getOperand(0);
2432 SDValue LHS = Op.getOperand(2);
2433 SDValue RHS = Op.getOperand(3);
2434 SDValue Dest = Op.getOperand(4);
2443 SDValue CompareFlag;
2464 static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG,
2464 static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG,
2467 SDValue LHS = Op.getOperand(0);
2468 SDValue RHS = Op.getOperand(1);
2470 SDValue TrueVal = Op.getOperand(2);
2471 SDValue FalseVal = Op.getOperand(3);
2479 SDValue CompareFlag;
2500 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG,
2500 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG,
2512 SDValue Offset =
2520 static SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) {
2520 static SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) {
2523 SDValue InChain = Node->getOperand(0);
2524 SDValue VAListPtr = Node->getOperand(1);
2528 SDValue VAList =
2531 SDValue NextPtr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
2543 static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG,
2543 static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG,
2545 SDValue Chain = Op.getOperand(0); // Legalize the chain.
2546 SDValue Size = Op.getOperand(1); // Legalize the size.
2594 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
2595 SDValue NewSP = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
2600 SDValue NewVal = DAG.getNode(ISD::ADD, dl, VT, NewSP,
2602 SDValue Ops[2] = { NewVal, Chain };
2607 static SDValue getFLUSHW(SDValue Op, SelectionDAG &DAG) {
2607 static SDValue getFLUSHW(SDValue Op, SelectionDAG &DAG) {
2609 SDValue Chain = DAG.getNode(SPISD::FLUSHW,
2614 static SDValue getFRAMEADDR(uint64_t depth, SDValue Op, SelectionDAG &DAG,
2614 static SDValue getFRAMEADDR(uint64_t depth, SDValue Op, SelectionDAG &DAG,
2625 SDValue FrameAddr;
2626 SDValue Chain;
2636 SDValue Ptr = DAG.getNode(ISD::ADD, dl, VT, FrameAddr,
2647 static SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG,
2647 static SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG,
2656 static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
2656 static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
2670 SDValue RetAddr;
2679 SDValue FrameAddr = getFRAMEADDR(depth - 1, Op, DAG, Subtarget, true);
2682 SDValue Ptr = DAG.getNode(ISD::ADD,
2691 static SDValue LowerF64Op(SDValue SrcReg64, const SDLoc &dl, SelectionDAG &DAG,
2691 static SDValue LowerF64Op(SDValue SrcReg64, const SDLoc &dl, SelectionDAG &DAG,
2705 SDValue Hi32 = DAG.getTargetExtractSubreg(SP::sub_even, dl, MVT::f32,
2707 SDValue Lo32 = DAG.getTargetExtractSubreg(SP::sub_odd, dl, MVT::f32,
2715 SDValue DstReg64 = SDValue(DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF,
2725 static SDValue LowerF128Load(SDValue Op, SelectionDAG &DAG)
2725 static SDValue LowerF128Load(SDValue Op, SelectionDAG &DAG)
2736 SDValue Hi64 =
2740 SDValue LoPtr = DAG.getNode(ISD::ADD, dl, addrVT,
2743 SDValue Lo64 = DAG.getLoad(MVT::f64, dl, LdNode->getChain(), LoPtr,
2746 SDValue SubRegEven = DAG.getTargetConstant(SP::sub_even64, dl, MVT::i32);
2747 SDValue SubRegOdd = DAG.getTargetConstant(SP::sub_odd64, dl, MVT::i32);
2761 SDValue OutChains[2] = { SDValue(Hi64.getNode(), 1),
2763 SDValue OutChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
2764 SDValue Ops[2] = {SDValue(InFP128,0), OutChain};
2768 static SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG)
2768 static SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG)
2780 static SDValue LowerF128Store(SDValue Op, SelectionDAG &DAG) {
2780 static SDValue LowerF128Store(SDValue Op, SelectionDAG &DAG) {
2785 SDValue SubRegEven = DAG.getTargetConstant(SP::sub_even64, dl, MVT::i32);
2786 SDValue SubRegOdd = DAG.getTargetConstant(SP::sub_odd64, dl, MVT::i32);
2803 SDValue OutChains[2];
2808 SDValue LoPtr = DAG.getNode(ISD::ADD, dl, addrVT,
2816 static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG)
2816 static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG)
2828 SDValue Val = DAG.getNode(ISD::BITCAST, dl, MVT::v2i32, St->getValue());
2829 SDValue Chain = DAG.getStore(
2838 static SDValue LowerFNEGorFABS(SDValue Op, SelectionDAG &DAG, bool isV9) {
2838 static SDValue LowerFNEGorFABS(SDValue Op, SelectionDAG &DAG, bool isV9) {
2854 SDValue SrcReg128 = Op.getOperand(0);
2855 SDValue Hi64 = DAG.getTargetExtractSubreg(SP::sub_even64, dl, MVT::f64,
2857 SDValue Lo64 = DAG.getTargetExtractSubreg(SP::sub_odd64, dl, MVT::f64,
2872 SDValue DstReg128 = SDValue(DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF,
2881 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2881 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2887 SDValue Src1 = Op.getOperand(0);
2888 SDValue Src1Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src1);
2889 SDValue Src1Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Src1,
2893 SDValue Src2 = Op.getOperand(1);
2894 SDValue Src2Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src2);
2895 SDValue Src2Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Src2,
2909 SDValue Lo;
2917 SDValue Hi = DAG.getNode(hiOpc, dl, VTs, Src1Hi, Src2Hi, Lo.getValue(1));
2918 SDValue Carry = Hi.getValue(1);
2925 SDValue Dst = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, Lo);
2926 SDValue Ops[2] = { Dst, Carry };
2932 static SDValue LowerUMULO_SMULO(SDValue Op, SelectionDAG &DAG,
2932 static SDValue LowerUMULO_SMULO(SDValue Op, SelectionDAG &DAG,
2942 SDValue LHS = Op.getOperand(0);
2947 SDValue ShiftAmt = DAG.getConstant(63, dl, VT);
2949 SDValue RHS = Op.getOperand(1);
2950 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, LHS, ShiftAmt);
2951 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, MVT::i64, RHS, ShiftAmt);
2952 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
2956 SDValue MulResult = TLI.makeLibCall(DAG,
2959 SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT,
2961 SDValue TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT,
2964 SDValue Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
2976 SDValue Ops[2] = { BottomHalf, TopHalf } ;
2980 static SDValue LowerATOMIC_LOAD_STORE(SDValue Op, SelectionDAG &DAG) {
2980 static SDValue LowerATOMIC_LOAD_STORE(SDValue Op, SelectionDAG &DAG) {
2989 SDValue SparcTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2989 SDValue SparcTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
3002 SDValue SparcTargetLowering::
3003 LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3064 SDValue SparcTargetLowering::bitcastConstantFPToInt(ConstantFPSDNode *C,
3068 SDValue Lo = DAG.getConstant(V.zextOrTrunc(32), DL, MVT::i32);
3069 SDValue Hi = DAG.getConstant(V.lshr(32).zextOrTrunc(32), DL, MVT::i32);
3075 SDValue SparcTargetLowering::PerformBITCASTCombine(SDNode *N,
3078 SDValue Src = N->getOperand(0);
3087 SDValue SparcTargetLowering::PerformDAGCombine(SDNode *N,
3223 LowerAsmOperandForConstraint(SDValue Op,
3225 std::vector<SDValue> &Ops,
3227 SDValue Result(nullptr, 0);
3335 SmallVectorImpl<SDValue>& Results,
3363 SDValue Lo = DAG.getCopyFromReg(N->getOperand(0), dl, SP::ASR23, MVT::i32);
3364 SDValue Hi = DAG.getCopyFromReg(Lo, dl, SP::G0, MVT::i32);
3365 SDValue Ops[] = { Lo, Hi };
3366 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops);
3395 SDValue LoadRes = DAG.getExtLoad(
3400 SDValue Res = DAG.getNode(ISD::BITCAST, dl, MVT::i64, LoadRes);
lib/Target/Sparc/SparcISelLowering.h 57 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
57 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
64 void computeKnownBitsForTargetNode(const SDValue Op,
80 void LowerAsmOperandForConstraint(SDValue Op,
82 std::vector<SDValue> &Ops,
126 SDValue
127 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
130 SmallVectorImpl<SDValue> &InVals) const override;
131 SDValue LowerFormalArguments_32(SDValue Chain, CallingConv::ID CallConv,
131 SDValue LowerFormalArguments_32(SDValue Chain, CallingConv::ID CallConv,
135 SmallVectorImpl<SDValue> &InVals) const;
136 SDValue LowerFormalArguments_64(SDValue Chain, CallingConv::ID CallConv,
136 SDValue LowerFormalArguments_64(SDValue Chain, CallingConv::ID CallConv,
140 SmallVectorImpl<SDValue> &InVals) const;
142 SDValue
144 SmallVectorImpl<SDValue> &InVals) const override;
145 SDValue LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
146 SmallVectorImpl<SDValue> &InVals) const;
147 SDValue LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
148 SmallVectorImpl<SDValue> &InVals) const;
150 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
150 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
152 const SmallVectorImpl<SDValue> &OutVals,
154 SDValue LowerReturn_32(SDValue Chain, CallingConv::ID CallConv,
154 SDValue LowerReturn_32(SDValue Chain, CallingConv::ID CallConv,
157 const SmallVectorImpl<SDValue> &OutVals,
159 SDValue LowerReturn_64(SDValue Chain, CallingConv::ID CallConv,
159 SDValue LowerReturn_64(SDValue Chain, CallingConv::ID CallConv,
162 const SmallVectorImpl<SDValue> &OutVals,
165 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
165 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
166 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
166 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
167 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
167 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
168 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
168 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
170 SDValue withTargetFlags(SDValue Op, unsigned TF, SelectionDAG &DAG) const;
170 SDValue withTargetFlags(SDValue Op, unsigned TF, SelectionDAG &DAG) const;
171 SDValue makeHiLoPair(SDValue Op, unsigned HiTF, unsigned LoTF,
171 SDValue makeHiLoPair(SDValue Op, unsigned HiTF, unsigned LoTF,
173 SDValue makeAddress(SDValue Op, SelectionDAG &DAG) const;
173 SDValue makeAddress(SDValue Op, SelectionDAG &DAG) const;
175 SDValue LowerF128_LibCallArg(SDValue Chain, ArgListTy &Args, SDValue Arg,
175 SDValue LowerF128_LibCallArg(SDValue Chain, ArgListTy &Args, SDValue Arg,
175 SDValue LowerF128_LibCallArg(SDValue Chain, ArgListTy &Args, SDValue Arg,
177 SDValue LowerF128Op(SDValue Op, SelectionDAG &DAG,
177 SDValue LowerF128Op(SDValue Op, SelectionDAG &DAG,
180 SDValue LowerF128Compare(SDValue LHS, SDValue RHS, unsigned &SPCC,
180 SDValue LowerF128Compare(SDValue LHS, SDValue RHS, unsigned &SPCC,
180 SDValue LowerF128Compare(SDValue LHS, SDValue RHS, unsigned &SPCC,
183 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
183 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
185 SDValue PerformBITCASTCombine(SDNode *N, DAGCombinerInfo &DCI) const;
187 SDValue bitcastConstantFPToInt(ConstantFPSDNode *C, const SDLoc &DL,
190 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
208 SmallVectorImpl<SDValue>& Results,
lib/Target/SystemZ/SystemZISelDAGToDAG.cpp 59 SDValue Base;
61 SDValue Index;
120 RxSBGOperands(unsigned Op, SDValue N)
128 SDValue Input;
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
159 SDValue &Base, SDValue &Disp) const;
159 SDValue &Base, SDValue &Disp) const;
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
167 SDValue &Base, SDValue &Disp) const;
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
173 SDValue &Base, SDValue &Disp) const;
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
268 SDValue &Disp, SDValue &Index) const;
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
342 SDValue expandSelectBoolean(SDNode *Node);
365 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
366 std::vector<SDValue> &OutOps) override;
367 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
402 SDValue Value) {
413 SDValue Value) {
424 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
425 SDValue Index) {
437 SDValue Op0, uint64_t Op1) {
453 SDValue N = IsBase ? AM.Base : AM.Index;
460 SDValue Op0 = N.getOperand(0);
461 SDValue Op1 = N.getOperand(1);
482 SDValue Full = N.getOperand(0);
483 SDValue Base = N.getOperand(1);
484 SDValue Anchor = Base.getOperand(0);
566 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
609 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
624 EVT VT, SDValue &Base,
625 SDValue &Disp) const {
639 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
649 EVT VT, SDValue &Base,
650 SDValue &Disp,
651 SDValue &Index) const {
661 SDValue Addr, SDValue &Base,
661 SDValue Addr, SDValue &Base,
662 SDValue &Disp) const {
672 SDValue Addr, SDValue &Base,
672 SDValue Addr, SDValue &Base,
673 SDValue &Disp) const {
684 SDValue Addr, SDValue &Base,
684 SDValue Addr, SDValue &Base,
685 SDValue &Disp, SDValue &Index) const {
685 SDValue &Disp, SDValue &Index) const {
694 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
694 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
695 SDValue &Base,
696 SDValue &Disp,
697 SDValue &Index) const {
698 SDValue Regs[2];
718 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
770 SDValue N = RxSBG.Input;
791 SDValue Input = N.getOperand(0);
814 SDValue Input = N.getOperand(0);
935 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
940 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
941 SDValue N) const {
1003 SDValue In = convertTo(DL, VT, RISBG.Input);
1004 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1005 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1038 SDValue Ops[5] = {
1045 SDValue New = convertTo(
1078 SDValue Op0 = N->getOperand(I ^ 1);
1095 SDValue Ops[5] = {
1102 SDValue New = convertTo(
1109 SDValue Op0, uint64_t UpperVal,
1113 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1135 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1136 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1152 SmallVector<SDValue, 2> Ops;
1155 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1160 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1173 SDValue ElemV = N->getOperand(2);
1190 SDValue Base, Disp, Index;
1196 SDValue Ops[] = {
1207 SDValue Value = Store->getValue();
1213 SDValue ElemV = Value.getOperand(1);
1218 SDValue Vec = Value.getOperand(0);
1224 SDValue Base, Disp, Index;
1230 SDValue Ops[] = {
1241 SDValue StoredVal, SelectionDAG *CurDAG,
1243 SDValue &InputChain) {
1256 SDValue Load = StoredVal->getOperand(0);
1275 SDValue Chain = StoreNode->getChain();
1282 SmallVector<SDValue, 4> ChainOps;
1287 SDValue Op = Chain.getOperand(i);
1300 for (SDValue Op : StoredVal->ops())
1331 SDValue StoredVal = StoreNode->getOperand(1);
1369 SDValue InputChain;
1374 SDValue Operand = StoredVal.getOperand(1);
1385 SDValue Base, Disp;
1389 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1499 auto Op0 = Node->getOperand(0);
1540 SDValue Op0 = Node->getOperand(0);
1541 SDValue Op1 = Node->getOperand(1);
1551 SDValue CCValid = Node->getOperand(2);
1552 SDValue CCMask = Node->getOperand(3);
1560 SDValue Op4 = Node->getOperand(4);
1626 SelectInlineAsmMemoryOperand(const SDValue &Op,
1628 std::vector<SDValue> &OutOps) {
1631 SDValue Base, Disp, Index;
1668 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1700 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1823 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1841 SDValue CCReg = Node->getOperand(4);
1843 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1897 SDValue Res;
lib/Target/SystemZ/SystemZISelLowering.cpp 35 Comparison(SDValue Op0In, SDValue Op1In)
35 Comparison(SDValue Op0In, SDValue Op1In)
39 SDValue Op0, Op1;
1152 LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
1153 std::vector<SDValue> &Ops,
1240 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1241 CCValAssign &VA, SDValue Chain,
1242 SDValue Value) {
1269 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1270 CCValAssign &VA, SDValue Value) {
1293 SDValue SystemZTargetLowering::LowerFormalArguments(
1294 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1296 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1318 SDValue ArgValue;
1367 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1387 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1416 SDValue MemOps[SystemZ::NumArgFPRs];
1420 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1423 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
1458 SDValue
1460 SmallVectorImpl<SDValue> &InVals) const {
1464 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1466 SDValue Chain = CLI.Chain;
1467 SDValue Callee = CLI.Callee;
1498 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
1499 SmallVector<SDValue, 8> MemOpChains;
1500 SDValue StackPtr;
1503 SDValue ArgValue = OutVals[I];
1507 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT);
1517 SDValue PartValue = OutVals[I + 1];
1519 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
1543 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
1559 SDValue Glue;
1580 SmallVector<SDValue, 8> Ops;
1624 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
1657 SDValue
1658 SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1661 const SmallVectorImpl<SDValue> &OutVals,
1679 SDValue Glue;
1680 SmallVector<SDValue, 4> RetOps;
1684 SDValue RetValue = OutVals[I];
1710 static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
1737 static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
1895 static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
1899 SmallVector<SDValue, 6> Ops;
1907 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
1908 SDValue OldChain = SDValue(Op.getNode(), 1);
1909 SDValue NewChain = SDValue(Intr.getNode(), 1);
1915 static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
1919 SmallVector<SDValue, 6> Ops;
1924 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);
2051 static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
2191 SDValue ShlOp0 = C.Op0.getOperand(0);
2228 static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
2449 SDValue Call, unsigned CCValid, uint64_t CC,
2481 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
2481 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
2536 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2566 SDValue Op0, SDValue Op1, SDValue &Hi,
2566 SDValue Op0, SDValue Op1, SDValue &Hi,
2566 SDValue Op0, SDValue Op1, SDValue &Hi,
2567 SDValue &Lo) {
2570 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
2582 unsigned Opcode, SDValue Op0, SDValue Op1,
2582 unsigned Opcode, SDValue Op0, SDValue Op1,
2583 SDValue &Even, SDValue &Odd) {
2583 SDValue &Even, SDValue &Odd) {
2584 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
2593 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
2593 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
2595 SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
2649 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
2650 SDValue Op) {
2658 SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
2660 SDValue CmpOp0,
2661 SDValue CmpOp1) const {
2667 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0);
2668 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0);
2669 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1);
2670 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1);
2671 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
2672 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
2680 SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
2683 SDValue CmpOp0,
2684 SDValue CmpOp1) const {
2687 SDValue Cmp;
2695 SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
2696 SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1);
2707 SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
2708 SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1);
2729 SDValue Mask =
2736 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
2736 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
2738 SDValue CmpOp0 = Op.getOperand(0);
2739 SDValue CmpOp1 = Op.getOperand(1);
2747 SDValue CCReg = emitCmp(DAG, DL, C);
2751 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
2751 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
2753 SDValue CmpOp0 = Op.getOperand(2);
2754 SDValue CmpOp1 = Op.getOperand(3);
2755 SDValue Dest = Op.getOperand(4);
2759 SDValue CCReg = emitCmp(DAG, DL, C);
2768 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
2768 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
2768 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
2779 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
2779 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
2788 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
2788 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
2790 SDValue CmpOp0 = Op.getOperand(0);
2791 SDValue CmpOp1 = Op.getOperand(1);
2792 SDValue TrueOp = Op.getOperand(2);
2793 SDValue FalseOp = Op.getOperand(3);
2813 SDValue CCReg = emitCmp(DAG, DL, C);
2814 SDValue Ops[] = {TrueOp, FalseOp,
2821 SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
2829 SDValue Result;
2839 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
2859 SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
2862 SDValue GOTOffset) const {
2865 SDValue Chain = DAG.getEntryNode();
2866 SDValue Glue;
2869 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
2876 SmallVector<SDValue, 8> Ops;
2906 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
2908 SDValue Chain = DAG.getEntryNode();
2912 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
2916 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
2920 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
2925 SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
2934 SDValue TP = lowerThreadPointer(DL, DAG);
2937 SDValue Offset;
2977 SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, 8);
3014 SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
3021 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
3026 SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
3030 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
3036 SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
3041 SDValue Result;
3053 SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
3053 SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
3071 SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
3081 SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
3081 SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
3104 SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
3104 SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
3107 SDValue In = Op.getOperand(0);
3116 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
3124 SDValue In64;
3135 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
3141 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3143 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
3147 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
3154 SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
3154 SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
3161 SDValue Chain = Op.getOperand(0);
3162 SDValue Addr = Op.getOperand(1);
3168 SDValue Fields[NumFields] = {
3176 SDValue MemOps[NumFields];
3179 SDValue FieldAddr = Addr;
3190 SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
3190 SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
3192 SDValue Chain = Op.getOperand(0);
3193 SDValue DstPtr = Op.getOperand(1);
3194 SDValue SrcPtr = Op.getOperand(2);
3205 SDValue SystemZTargetLowering::
3206 lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
3212 SDValue Chain = Op.getOperand(0);
3213 SDValue Size = Op.getOperand(1);
3214 SDValue Align = Op.getOperand(2);
3227 SDValue NeededSpace = Size;
3230 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
3233 SDValue Backchain;
3243 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
3251 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
3252 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
3267 SDValue Ops[2] = { Result, Chain };
3271 SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
3272 SDValue Op, SelectionDAG &DAG) const {
3278 SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
3278 SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
3282 SDValue Ops[2];
3308 SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
3309 SDValue LL = Op.getOperand(0);
3310 SDValue RL = Op.getOperand(1);
3311 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
3312 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
3318 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
3319 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
3320 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
3326 SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
3326 SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
3330 SDValue Ops[2];
3345 SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
3345 SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
3347 SDValue Op0 = Op.getOperand(0);
3348 SDValue Op1 = Op.getOperand(1);
3362 SDValue Ops[2];
3367 SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
3367 SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
3374 SDValue Ops[2];
3380 SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
3380 SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
3384 SDValue Ops[] = {Op.getOperand(0), Op.getOperand(1)};
3400 SDValue LowOp = Ops[Low];
3401 SDValue HighOp = Ops[High];
3419 SDValue HighOp0 = HighOp.getOperand(0);
3430 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
3436 SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
3436 SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
3439 SDValue LHS = N->getOperand(0);
3440 SDValue RHS = N->getOperand(1);
3471 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
3473 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
3480 static bool isAddCarryChain(SDValue Carry) {
3486 static bool isSubBorrowChain(SDValue Carry) {
3493 SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
3493 SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
3503 SDValue LHS = N->getOperand(0);
3504 SDValue RHS = N->getOperand(1);
3505 SDValue Carry = Op.getOperand(2);
3537 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
3539 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
3546 SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
3546 SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
3561 SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
3562 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
3568 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
3574 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
3605 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
3620 SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
3620 SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
3642 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
3642 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
3651 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
3651 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
3654 SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
3667 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
3667 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
3679 SDValue ChainIn = Node->getChain();
3680 SDValue Addr = Node->getBasePtr();
3681 SDValue Src2 = Node->getVal();
3694 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3699 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3705 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3723 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
3725 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
3730 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
3732 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
3734 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
3741 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
3741 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
3748 SDValue Src2 = Node->getVal();
3749 SDValue NegSrc2;
3776 SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
3776 SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
3779 SDValue ChainIn = Node->getOperand(0);
3780 SDValue Addr = Node->getOperand(1);
3781 SDValue CmpVal = Node->getOperand(2);
3782 SDValue SwapVal = Node->getOperand(3);
3792 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
3793 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
3795 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
3810 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3815 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3821 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3826 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
3828 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
3830 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
3860 SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
3860 SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
3868 SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
3868 SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
3874 SDValue Chain = Op.getOperand(0);
3875 SDValue NewSP = Op.getOperand(1);
3876 SDValue Backchain;
3880 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, MVT::i64);
3892 SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
3892 SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
3903 SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
3911 static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
3911 static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
3913 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
3918 SDValue
3919 SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
3925 SDValue CC = getCCResult(DAG, SDValue(Node, 0));
3933 SDValue
3934 SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
4149 static bool getVPermMask(SDValue ShuffleOp,
4232 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
4233 const Permute &P, SDValue Op0, SDValue Op1) {
4233 const Permute &P, SDValue Op0, SDValue Op1) {
4244 SDValue Op;
4246 SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
4261 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
4262 SDValue *Ops,
4275 SDValue IndexNodes[SystemZ::VectorBytes];
4281 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
4290 bool add(SDValue, unsigned);
4291 SDValue getNode(SelectionDAG &, const SDLoc &);
4294 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
4319 bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
4379 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
4401 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
4449 SDValue Op;
4458 static bool isScalarToVector(SDValue Op) {
4467 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4468 SDValue Value) {
4473 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
4483 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4484 SDValue Op0, SDValue Op1) {
4484 SDValue Op0, SDValue Op1) {
4499 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
4499 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
4500 SDValue Op1) {
4520 static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
4530 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
4533 SDValue Op = BVN->getOperand(I);
4559 for (auto &Op : GS.Ops) {
4569 bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
4578 SDValue
4580 SmallVectorImpl<SDValue> &Elems) const {
4582 SDValue Single;
4585 for (auto Elem : Elems) {
4614 for (auto Elem : Elems)
4636 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4637 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
4648 SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH,
4654 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
4659 SDValue Elem = Elems[I];
4670 SDValue Result;
4671 SDValue ReplicatedVal;
4703 SDValue Elem1 = Elems[Def1 ? I1 : I2];
4704 SDValue Elem2 = Elems[Def2 ? I2 : I1];
4722 SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
4722 SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
4737 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
4746 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
4752 SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
4752 SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
4760 SDValue Op0 = Op.getOperand(0);
4785 SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
4785 SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
4794 SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4794 SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4798 SDValue Op0 = Op.getOperand(0);
4799 SDValue Op1 = Op.getOperand(1);
4800 SDValue Op2 = Op.getOperand(2);
4819 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
4825 SDValue
4826 SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4830 SDValue Op0 = Op.getOperand(0);
4831 SDValue Op1 = Op.getOperand(1);
4846 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
4851 SDValue
4852 SystemZTargetLowering::lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
4854 SDValue PackedOp = Op.getOperand(0);
4868 SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
4868 SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
4871 SDValue Op0 = Op.getOperand(0);
4872 SDValue Op1 = Op.getOperand(1);
4887 SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
4893 SDValue Splat = BVN->getSplatValue(&UndefElements);
4897 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
4906 SDValue VSNOp0 = VSN->getOperand(0);
4914 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
4925 SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
4925 SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
5046 static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
5046 static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
5048 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
5050 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
5057 static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
5057 static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
5059 SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
5061 SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
5068 SmallVectorImpl<SDValue> &Results,
5074 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
5076 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_LOAD_128,
5085 SDValue Ops[] = { N->getOperand(0),
5089 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
5103 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
5107 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
5109 SDValue Success = emitSETCC(DAG, DL, Res.getValue(1),
5124 SmallVectorImpl<SDValue> &Results,
5266 SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
5267 EVT VecVT, SDValue Op,
5372 SDValue SystemZTargetLowering::combineTruncateExtract(
5373 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
5379 SDValue Vec = Op.getOperand(0);
5407 SDValue SystemZTargetLowering::combineZERO_EXTEND(
5411 SDValue N0 = N->getOperand(0);
5418 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
5421 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
5424 SDValue TruncSelect =
5434 SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
5440 SDValue N0 = N->getOperand(0);
5447 SDValue Ops[] = { N0.getOperand(0), N0.getOperand(1),
5455 SDValue SystemZTargetLowering::combineSIGN_EXTEND(
5461 SDValue N0 = N->getOperand(0);
5465 SDValue Inner = N0.getOperand(0);
5472 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
5474 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
5485 SDValue SystemZTargetLowering::combineMERGE(
5489 SDValue Op0 = N->getOperand(0);
5490 SDValue Op1 = N->getOperand(1);
5511 SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
5519 SDValue SystemZTargetLowering::combineLOAD(
5531 SDValue Replicate;
5547 SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT,
5551 SmallVector<SDValue, 8> Ops;
5552 for (SDValue Op : U->ops())
5584 SDValue SystemZTargetLowering::combineSTORE(
5595 if (SDValue Value =
5611 SDValue BSwapOp = Op1.getOperand(0);
5616 SDValue Ops[] = {
5632 SDValue Ops[] = {
5645 SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
5655 SDValue Load = N->getOperand(0);
5659 SDValue Ops[] = {
5663 SDValue ESLoad =
5684 SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
5692 SDValue Op = N->getOperand(0);
5717 SDValue Op0 = N->getOperand(0);
5725 SDValue SystemZTargetLowering::combineJOIN_DWORDS(
5735 SDValue SystemZTargetLowering::combineFP_ROUND(
5748 SDValue Op0 = N->getOperand(0);
5755 SDValue Vec = Op0.getOperand(0);
5763 SDValue OtherRound = SDValue(*U->use_begin(), 0);
5767 SDValue VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
5770 SDValue Extract1 =
5775 SDValue Extract0 =
5786 SDValue SystemZTargetLowering::combineFP_EXTEND(
5799 SDValue Op0 = N->getOperand(0);
5806 SDValue Vec = Op0.getOperand(0);
5814 SDValue OtherExtend = SDValue(*U->use_begin(), 0);
5818 SDValue VExtend = DAG.getNode(SystemZISD::VEXTEND, SDLoc(N),
5821 SDValue Extract1 =
5826 SDValue Extract0 =
5837 SDValue SystemZTargetLowering::combineBSWAP(
5844 SDValue Load = N->getOperand(0);
5848 SDValue Ops[] = {
5855 SDValue BSLoad =
5861 SDValue ResVal = BSLoad;
5878 SDValue Op = N->getOperand(0);
5888 SDValue Vec = Op.getOperand(0);
5889 SDValue Elt = Op.getOperand(1);
5890 SDValue Idx = Op.getOperand(2);
5920 SDValue Op0 = Op.getOperand(0);
5921 SDValue Op1 = Op.getOperand(1);
5947 static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
6042 SDValue SystemZTargetLowering::combineBR_CCMASK(
6054 SDValue Chain = N->getOperand(0);
6055 SDValue CCReg = N->getOperand(4);
6066 SDValue SystemZTargetLowering::combineSELECT_CCMASK(
6078 SDValue CCReg = N->getOperand(4);
6090 SDValue SystemZTargetLowering::combineGET_CCMASK(
6101 SDValue Select = N->getOperand(0);
6131 SDValue SystemZTargetLowering::combineIntDIVREM(
6148 SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
6148 SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
6154 SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
6185 static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
6282 static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
6297 SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
6355 SDValue SrcOp = Op.getOperand(1);
6374 SDValue SrcOp = Op.getOperand(0);
6391 static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
6416 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6446 SDValue PackedOp = Op.getOperand(1);
lib/Target/SystemZ/SystemZISelLowering.h 430 void LowerAsmOperandForConstraint(SDValue Op,
432 std::vector<SDValue> &Ops,
479 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
479 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
480 void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results,
482 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
487 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
487 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
491 SmallVectorImpl<SDValue> &InVals) const override;
492 SDValue LowerCall(CallLoweringInfo &CLI,
493 SmallVectorImpl<SDValue> &InVals) const override;
499 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
499 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
501 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
503 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
507 void computeKnownBitsForTargetNode(const SDValue Op,
514 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
531 SDValue getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
533 SDValue CmpOp0, SDValue CmpOp1) const;
533 SDValue CmpOp0, SDValue CmpOp1) const;
534 SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL,
536 SDValue CmpOp0, SDValue CmpOp1) const;
536 SDValue CmpOp0, SDValue CmpOp1) const;
537 SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
537 SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
538 SDValue lowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
538 SDValue lowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
539 SDValue lowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
539 SDValue lowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
540 SDValue lowerGlobalAddress(GlobalAddressSDNode *Node,
542 SDValue lowerTLSGetOffset(GlobalAddressSDNode *Node,
544 SDValue GOTOffset) const;
545 SDValue lowerThreadPointer(const SDLoc &DL, SelectionDAG &DAG) const;
546 SDValue lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
548 SDValue lowerBlockAddress(BlockAddressSDNode *Node,
550 SDValue lowerJumpTable(JumpTableSDNode *JT, SelectionDAG &DAG) const;
551 SDValue lowerConstantPool(ConstantPoolSDNode *CP, SelectionDAG &DAG) const;
552 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
552 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
553 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
553 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
554 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
554 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
555 SDValue lowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
555 SDValue lowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
556 SDValue lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
556 SDValue lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
557 SDValue lowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const;
557 SDValue lowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const;
558 SDValue lowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
558 SDValue lowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
559 SDValue lowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
559 SDValue lowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
560 SDValue lowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
560 SDValue lowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
561 SDValue lowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
561 SDValue lowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
562 SDValue lowerXALUO(SDValue Op, SelectionDAG &DAG) const;
562 SDValue lowerXALUO(SDValue Op, SelectionDAG &DAG) const;
563 SDValue lowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) const;
563 SDValue lowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) const;
564 SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
564 SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
565 SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
565 SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
566 SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
566 SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
567 SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
567 SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
568 SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
568 SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
569 SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
569 SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
570 SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
570 SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
572 SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
572 SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
573 SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
573 SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
574 SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
574 SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
575 SDValue lowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
575 SDValue lowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
576 SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
576 SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
577 SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
577 SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
578 SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
578 SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
579 bool isVectorElementLoad(SDValue Op) const;
580 SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
581 SmallVectorImpl<SDValue> &Elems) const;
582 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
582 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
583 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
583 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
584 SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
584 SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
585 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
585 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
586 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
586 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
587 SDValue lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
587 SDValue lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
589 SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
589 SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
592 SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp,
592 SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp,
595 SDValue combineTruncateExtract(const SDLoc &DL, EVT TruncVT, SDValue Op,
595 SDValue combineTruncateExtract(const SDLoc &DL, EVT TruncVT, SDValue Op,
597 SDValue combineZERO_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
598 SDValue combineSIGN_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
599 SDValue combineSIGN_EXTEND_INREG(SDNode *N, DAGCombinerInfo &DCI) const;
600 SDValue combineMERGE(SDNode *N, DAGCombinerInfo &DCI) const;
602 SDValue combineLOAD(SDNode *N, DAGCombinerInfo &DCI) const;
603 SDValue combineSTORE(SDNode *N, DAGCombinerInfo &DCI) const;
604 SDValue combineVECTOR_SHUFFLE(SDNode *N, DAGCombinerInfo &DCI) const;
605 SDValue combineEXTRACT_VECTOR_ELT(SDNode *N, DAGCombinerInfo &DCI) const;
606 SDValue combineJOIN_DWORDS(SDNode *N, DAGCombinerInfo &DCI) const;
607 SDValue combineFP_ROUND(SDNode *N, DAGCombinerInfo &DCI) const;
608 SDValue combineFP_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
609 SDValue combineBSWAP(SDNode *N, DAGCombinerInfo &DCI) const;
610 SDValue combineBR_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
611 SDValue combineSELECT_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
612 SDValue combineGET_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
613 SDValue combineIntDIVREM(SDNode *N, DAGCombinerInfo &DCI) const;
615 SDValue unwrapAddress(SDValue N) const override;
615 SDValue unwrapAddress(SDValue N) const override;
lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp 25 static SDValue emitMemMem(SelectionDAG &DAG, const SDLoc &DL, unsigned Sequence,
26 unsigned Loop, SDValue Chain, SDValue Dst,
26 unsigned Loop, SDValue Chain, SDValue Dst,
27 SDValue Src, uint64_t Size) {
48 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
49 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
49 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
49 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
50 SDValue Size, unsigned Align, bool IsVolatile, bool AlwaysInline,
64 static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
64 static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
65 SDValue Dst, uint64_t ByteVal, uint64_t Size,
75 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
76 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
76 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
77 SDValue Byte, SDValue Size, unsigned Align, bool IsVolatile,
77 SDValue Byte, SDValue Size, unsigned Align, bool IsVolatile,
99 SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1,
106 SDValue Chain2 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size2,
113 SDValue Chain1 = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Align);
116 SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
118 SDValue Chain2 =
135 SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
145 static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
145 static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
146 SDValue Src1, SDValue Src2, uint64_t Size) {
146 SDValue Src1, SDValue Src2, uint64_t Size) {
169 static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
169 static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
171 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
172 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, IPM,
174 SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL,
179 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
179 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
180 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
180 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
181 SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
181 SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
187 SDValue CCReg = emitCLC(DAG, DL, Chain, Src2, Src1, Bytes);
194 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
194 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
195 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
195 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
196 SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
196 SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
204 SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, Length);
205 SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
207 SDValue CCReg = End.getValue(1);
212 SDValue Ops[] = {
220 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy(
220 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy(
221 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
221 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
222 SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo,
225 SDValue EndDest = DAG.getNode(SystemZISD::STPCPY, DL, VTs, Chain, Dest, Src,
230 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
230 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
231 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
231 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
232 SDValue Src2, MachinePointerInfo Op1PtrInfo,
236 SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src2, Src1,
238 SDValue CCReg = Unused.getValue(1);
248 static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG,
248 static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG,
250 SDValue Chain, SDValue Src,
250 SDValue Chain, SDValue Src,
251 SDValue Limit) {
254 SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain,
257 SDValue Len = DAG.getNode(ISD::SUB, DL, PtrVT, End, Src);
261 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
261 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
262 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
262 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
268 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen(
268 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen(
269 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
269 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
270 SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const {
273 SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, MaxLength);
lib/Target/SystemZ/SystemZSelectionDAGInfo.h 26 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &DL,
27 SDValue Chain, SDValue Dst, SDValue Src,
27 SDValue Chain, SDValue Dst, SDValue Src,
27 SDValue Chain, SDValue Dst, SDValue Src,
28 SDValue Size, unsigned Align, bool IsVolatile,
33 SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &DL,
34 SDValue Chain, SDValue Dst, SDValue Byte,
34 SDValue Chain, SDValue Dst, SDValue Byte,
34 SDValue Chain, SDValue Dst, SDValue Byte,
35 SDValue Size, unsigned Align, bool IsVolatile,
38 std::pair<SDValue, SDValue>
38 std::pair<SDValue, SDValue>
39 EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
40 SDValue Src1, SDValue Src2, SDValue Size,
40 SDValue Src1, SDValue Src2, SDValue Size,
40 SDValue Src1, SDValue Src2, SDValue Size,
44 std::pair<SDValue, SDValue>
44 std::pair<SDValue, SDValue>
45 EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
46 SDValue Src, SDValue Char, SDValue Length,
46 SDValue Src, SDValue Char, SDValue Length,
46 SDValue Src, SDValue Char, SDValue Length,
49 std::pair<SDValue, SDValue> EmitTargetCodeForStrcpy(
49 std::pair<SDValue, SDValue> EmitTargetCodeForStrcpy(
50 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
50 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
51 SDValue Src, MachinePointerInfo DestPtrInfo,
54 std::pair<SDValue, SDValue>
54 std::pair<SDValue, SDValue>
55 EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
56 SDValue Src1, SDValue Src2,
56 SDValue Src1, SDValue Src2,
60 std::pair<SDValue, SDValue>
60 std::pair<SDValue, SDValue>
61 EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
62 SDValue Src,
65 std::pair<SDValue, SDValue>
65 std::pair<SDValue, SDValue>
66 EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
67 SDValue Src, SDValue MaxLength,
67 SDValue Src, SDValue MaxLength,
lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp 68 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
69 std::vector<SDValue> &OutOps) override;
153 SDValue TLSBaseSym = CurDAG->getTargetExternalSymbol("__tls_base", PtrVT);
154 SDValue TLSOffsetSym = CurDAG->getTargetGlobalAddress(
221 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
221 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
lib/Target/WebAssembly/WebAssemblyISelLowering.cpp 551 bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
638 SDValue
640 SmallVectorImpl<SDValue> &InVals) const {
643 SDValue Chain = CLI.Chain;
644 SDValue Callee = CLI.Callee;
693 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
707 SDValue &OutVal = OutVals[I];
721 SDValue SizeNode =
723 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
746 SDValue &Arg = OutVals[I];
762 SDValue FINode;
770 SmallVector<SDValue, 8> Chains;
771 for (SDValue Arg :
777 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
802 SmallVector<SDValue, 16> Ops;
838 SDValue Res =
859 SDValue WebAssemblyTargetLowering::LowerReturn(
860 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
862 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
869 SmallVector<SDValue, 4> RetOps(1, Chain);
889 SDValue WebAssemblyTargetLowering::LowerFormalArguments(
890 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
892 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
954 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
972 SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
972 SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1021 SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1021 SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1023 SDValue Src = Op.getOperand(2);
1030 SDValue Chain = Op.getOperand(0);
1034 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1047 SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1047 SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1053 SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1053 SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1074 SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1074 SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1089 SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1089 SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1114 SDValue BaseAddr =
1118 SDValue SymAddr = DAG.getNode(
1134 SDValue
1135 WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1146 SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1146 SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1156 SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1156 SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1159 SDValue Chain = Op.getOperand(0);
1161 SDValue Index = Op.getOperand(2);
1164 SmallVector<SDValue, 8> Ops;
1183 SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1183 SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1191 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1197 SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1197 SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1237 SDValue SymNode = DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1250 SDValue
1251 WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1262 const SDValue &Extract = Op.getOperand(0);
1272 const SDValue &Index = Extract.getOperand(1);
1278 SDValue NewIndex =
1280 SDValue NewExtract = DAG.getNode(
1290 SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1290 SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1321 const SDValue &SwizzleSrc = Lane->getOperand(0);
1322 const SDValue &IndexExt = Lane->getOperand(1);
1325 const SDValue &Index = IndexExt->getOperand(0);
1328 const SDValue &SwizzleIndices = Index->getOperand(0);
1337 using ValueEntry = std::pair<SDValue, size_t>;
1340 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
1340 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
1365 const SDValue &Lane = Op->getOperand(I);
1380 SDValue SplatValue;
1384 SDValue SwizzleSrc;
1385 SDValue SwizzleIndices;
1393 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
1394 SDValue Result;
1406 SmallVector<SDValue, 16> ConstLanes;
1407 for (const SDValue &Lane : Op->op_values()) {
1439 const SDValue &Lane = Op->getOperand(I);
1448 SDValue
1449 WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
1458 SDValue Ops[18];
1475 SDValue
1476 WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
1487 static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
1487 static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
1494 SDValue ShiftVal = Op.getOperand(1);
1496 SDValue MaskedShiftVal = DAG.getNode(
1512 SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
1512 SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
1521 SDValue SplatVal;
lib/Target/WebAssembly/WebAssemblyISelLowering.h 66 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
73 SDValue LowerCall(CallLoweringInfo &CLI,
74 SmallVectorImpl<SDValue> &InVals) const override;
79 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
79 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
81 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
83 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
83 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
87 SmallVectorImpl<SDValue> &InVals) const override;
89 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
97 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
97 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
98 SDValue LowerFrameIndex(SDValue Op, SelectionDAG &DAG) const;
98 SDValue LowerFrameIndex(SDValue Op, SelectionDAG &DAG) const;
99 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
99 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
100 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
100 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
101 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
101 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
102 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
102 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
103 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
103 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
104 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
104 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
105 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
105 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
106 SDValue LowerCopyToReg(SDValue Op, SelectionDAG &DAG) const;
106 SDValue LowerCopyToReg(SDValue Op, SelectionDAG &DAG) const;
107 SDValue LowerIntrinsic(SDValue Op, SelectionDAG &DAG) const;
107 SDValue LowerIntrinsic(SDValue Op, SelectionDAG &DAG) const;
108 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
108 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
109 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
109 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
110 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
110 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
111 SDValue LowerAccessVectorElement(SDValue Op, SelectionDAG &DAG) const;
111 SDValue LowerAccessVectorElement(SDValue Op, SelectionDAG &DAG) const;
112 SDValue LowerShift(SDValue Op, SelectionDAG &DAG) const;
112 SDValue LowerShift(SDValue Op, SelectionDAG &DAG) const;
lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp 21 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(
22 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
22 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
22 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
23 SDValue Size, unsigned Align, bool IsVolatile, bool AlwaysInline,
30 SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
36 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemmove(
37 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op1, SDValue Op2,
37 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op1, SDValue Op2,
37 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op1, SDValue Op2,
38 SDValue Op3, unsigned Align, bool IsVolatile,
45 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset(
46 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Val,
46 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Val,
46 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Val,
47 SDValue Size, unsigned Align, bool IsVolatile,
54 SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h 25 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
26 SDValue Chain, SDValue Op1, SDValue Op2,
26 SDValue Chain, SDValue Op1, SDValue Op2,
26 SDValue Chain, SDValue Op1, SDValue Op2,
27 SDValue Op3, unsigned Align, bool isVolatile,
31 SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl,
32 SDValue Chain, SDValue Op1, SDValue Op2,
32 SDValue Chain, SDValue Op1, SDValue Op2,
32 SDValue Chain, SDValue Op1, SDValue Op2,
33 SDValue Op3, unsigned Align, bool isVolatile,
36 SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &DL,
37 SDValue Chain, SDValue Op1, SDValue Op2,
37 SDValue Chain, SDValue Op1, SDValue Op2,
37 SDValue Chain, SDValue Op1, SDValue Op2,
38 SDValue Op3, unsigned Align, bool IsVolatile,
lib/Target/X86/X86ISelDAGToDAG.cpp 61 SDValue Base_Reg;
65 SDValue IndexReg;
67 SDValue Segment;
102 void setBaseReg(SDValue Reg) {
200 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
213 bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
214 bool matchAddress(SDValue N, X86ISelAddressMode &AM);
215 bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
216 bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
217 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
219 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
220 bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
220 bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
221 SDValue &Scale, SDValue &Index, SDValue &Disp,
221 SDValue &Scale, SDValue &Index, SDValue &Disp,
221 SDValue &Scale, SDValue &Index, SDValue &Disp,
222 SDValue &Segment);
223 bool selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
223 bool selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
224 SDValue &Scale, SDValue &Index, SDValue &Disp,
224 SDValue &Scale, SDValue &Index, SDValue &Disp,
224 SDValue &Scale, SDValue &Index, SDValue &Disp,
225 SDValue &Segment);
226 bool selectMOV64Imm32(SDValue N, SDValue &Imm);
226 bool selectMOV64Imm32(SDValue N, SDValue &Imm);
227 bool selectLEAAddr(SDValue N, SDValue &Base,
227 bool selectLEAAddr(SDValue N, SDValue &Base,
228 SDValue &Scale, SDValue &Index, SDValue &Disp,
228 SDValue &Scale, SDValue &Index, SDValue &Disp,
228 SDValue &Scale, SDValue &Index, SDValue &Disp,
229 SDValue &Segment);
230 bool selectLEA64_32Addr(SDValue N, SDValue &Base,
230 bool selectLEA64_32Addr(SDValue N, SDValue &Base,
231 SDValue &Scale, SDValue &Index, SDValue &Disp,
231 SDValue &Scale, SDValue &Index, SDValue &Disp,
231 SDValue &Scale, SDValue &Index, SDValue &Disp,
232 SDValue &Segment);
233 bool selectTLSADDRAddr(SDValue N, SDValue &Base,
233 bool selectTLSADDRAddr(SDValue N, SDValue &Base,
234 SDValue &Scale, SDValue &Index, SDValue &Disp,
234 SDValue &Scale, SDValue &Index, SDValue &Disp,
234 SDValue &Scale, SDValue &Index, SDValue &Disp,
235 SDValue &Segment);
236 bool selectScalarSSELoad(SDNode *Root, SDNode *Parent, SDValue N,
237 SDValue &Base, SDValue &Scale,
237 SDValue &Base, SDValue &Scale,
238 SDValue &Index, SDValue &Disp,
238 SDValue &Index, SDValue &Disp,
239 SDValue &Segment,
240 SDValue &NodeWithChain);
241 bool selectRelocImm(SDValue N, SDValue &Op);
241 bool selectRelocImm(SDValue N, SDValue &Op);
243 bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
244 SDValue &Base, SDValue &Scale,
244 SDValue &Base, SDValue &Scale,
245 SDValue &Index, SDValue &Disp,
245 SDValue &Index, SDValue &Disp,
246 SDValue &Segment);
249 bool tryFoldLoad(SDNode *P, SDValue N,
250 SDValue &Base, SDValue &Scale,
250 SDValue &Base, SDValue &Scale,
251 SDValue &Index, SDValue &Disp,
251 SDValue &Index, SDValue &Disp,
252 SDValue &Segment) {
256 bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
257 SDValue &Base, SDValue &Scale,
257 SDValue &Base, SDValue &Scale,
258 SDValue &Index, SDValue &Disp,
258 SDValue &Index, SDValue &Disp,
259 SDValue &Segment);
262 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
264 std::vector<SDValue> &OutOps) override;
269 MVT VT, SDValue &Base, SDValue &Scale,
269 MVT VT, SDValue &Base, SDValue &Scale,
270 SDValue &Index, SDValue &Disp,
270 SDValue &Index, SDValue &Disp,
271 SDValue &Segment) {
285 SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32,
385 SDValue OtherOp = User->getOperand(0);
408 inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) {
413 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
418 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) {
422 SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth,
430 SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth,
517 bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
517 bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
524 SDValue &InFlag);
528 bool onlyUsesZeroFlag(SDValue Flags) const;
529 bool hasNoSignFlagUses(SDValue Flags) const;
530 bool hasNoCarryFlagUses(SDValue Flags) const;
572 X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
601 SDValue Op1 = U->getOperand(1);
654 SDValue Val = Op1.getOperand(0);
673 SDValue U0 = U->getOperand(0);
674 SDValue U1 = U->getOperand(1);
717 static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
718 SDValue Call, SDValue OrigChain) {
718 SDValue Call, SDValue OrigChain) {
719 SmallVector<SDValue, 8> Ops;
720 SDValue Chain = OrigChain.getOperand(0);
731 SDValue NewChain =
752 static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
752 static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
797 SDValue Res = CurDAG->getNode(ISD::AND, SDLoc(N), N->getValueType(0),
820 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
843 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
863 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
888 SDValue Res = CurDAG->getNode(
909 SDValue Op0 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT,
911 SDValue Op1 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT,
914 SDValue Res;
970 SDValue Chain = N->getOperand(0);
971 SDValue Load = N->getOperand(1);
1026 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1031 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, N->getOperand(0),
1033 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp,
1084 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1090 SDValue Store = CurDAG->getTruncStore(N->getOperand(0), dl, N->getOperand(1),
1092 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp,
1125 SDValue N0 = N->getOperand(0);
1136 SDValue N00 = N0.getOperand(0);
1181 SDValue And = N->getOperand(0);
1204 SDValue Ops[] = { And.getOperand(1),
1229 SDValue And = N->getOperand(0);
1262 SDValue Move = N->getOperand(1);
1287 SDValue In = Move.getOperand(0);
1323 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
1323 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
1373 SDValue Address = N->getOperand(1);
1403 bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) {
1412 SDValue Val = N.getOperand(0);
1436 SDValue N0 = N.getOperand(0);
1475 bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) {
1509 bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM,
1548 static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) {
1548 static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) {
1566 static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N,
1568 SDValue Shift, SDValue X,
1568 SDValue Shift, SDValue X,
1582 SDValue Eight = DAG.getConstant(8, DL, MVT::i8);
1583 SDValue NewMask = DAG.getConstant(0xff, DL, VT);
1584 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, X, Eight);
1585 SDValue And = DAG.getNode(ISD::AND, DL, VT, Srl, NewMask);
1586 SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8);
1587 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And, ShlCount);
1610 static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
1612 SDValue Shift = N.getOperand(0);
1634 SDValue X = Shift.getOperand(0);
1650 SDValue NewX = DAG.getNode(ISD::ANY_EXTEND, DL, VT, X);
1655 SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT);
1656 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask);
1657 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1));
1702 static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
1704 SDValue Shift, SDValue X,
1704 SDValue Shift, SDValue X,
1759 SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X);
1764 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8);
1765 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt);
1766 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8);
1767 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewSRL, NewSHLAmt);
1789 static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N,
1791 SDValue Shift, SDValue X,
1791 SDValue Shift, SDValue X,
1819 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8);
1820 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt);
1821 SDValue NewMask = DAG.getConstant(Mask >> AMShiftAmt, DL, VT);
1822 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, NewSRL, NewMask);
1823 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8);
1824 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewAnd, NewSHLAmt);
1845 bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
1923 SDValue ShVal = N.getOperand(0);
1951 SDValue And = N.getOperand(0);
1953 SDValue X = And.getOperand(0);
1986 SDValue MulVal = N.getOperand(0);
1987 SDValue Reg;
2037 SDValue RHS = N.getOperand(1);
2108 SDValue Shift = N.getOperand(0);
2109 SDValue X = Shift.getOperand(0);
2142 SDValue Shl = N.getOperand(0);
2157 SDValue Zext = CurDAG->getNode(ISD::ZERO_EXTEND, DL, VT, Shl.getOperand(0));
2158 SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, Shl.getOperand(1));
2177 bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) {
2200 bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) {
2218 bool X86DAGToDAGISel::selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
2218 bool X86DAGToDAGISel::selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
2219 SDValue &Scale, SDValue &Index,
2219 SDValue &Scale, SDValue &Index,
2220 SDValue &Disp, SDValue &Segment) {
2220 SDValue &Disp, SDValue &Segment) {
2253 bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
2253 bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
2254 SDValue &Scale, SDValue &Index,
2254 SDValue &Scale, SDValue &Index,
2255 SDValue &Disp, SDValue &Segment) {
2255 SDValue &Disp, SDValue &Segment) {
2311 SDValue N, SDValue &Base,
2311 SDValue N, SDValue &Base,
2312 SDValue &Scale, SDValue &Index,
2312 SDValue &Scale, SDValue &Index,
2313 SDValue &Disp, SDValue &Segment,
2313 SDValue &Disp, SDValue &Segment,
2314 SDValue &PatternNodeWithChain) {
2360 bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
2360 bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
2394 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base,
2394 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base,
2395 SDValue &Scale, SDValue &Index,
2395 SDValue &Scale, SDValue &Index,
2396 SDValue &Disp, SDValue &Segment) {
2396 SDValue &Disp, SDValue &Segment) {
2408 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
2420 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
2431 bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
2432 SDValue &Base, SDValue &Scale,
2432 SDValue &Base, SDValue &Scale,
2433 SDValue &Index, SDValue &Disp,
2433 SDValue &Index, SDValue &Disp,
2434 SDValue &Segment) {
2443 SDValue Copy = AM.Segment;
2444 SDValue T = CurDAG->getRegister(0, MVT::i32);
2521 bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base,
2521 bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base,
2522 SDValue &Scale, SDValue &Index,
2522 SDValue &Scale, SDValue &Index,
2523 SDValue &Disp, SDValue &Segment) {
2523 SDValue &Disp, SDValue &Segment) {
2542 bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) {
2542 bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) {
2585 bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
2586 SDValue &Base, SDValue &Scale,
2586 SDValue &Base, SDValue &Scale,
2587 SDValue &Index, SDValue &Disp,
2587 SDValue &Index, SDValue &Disp,
2588 SDValue &Segment) {
2599 bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
2600 SDValue &Base, SDValue &Scale,
2600 SDValue &Base, SDValue &Scale,
2601 SDValue &Index, SDValue &Disp,
2601 SDValue &Index, SDValue &Disp,
2602 SDValue &Segment) {
2659 bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const {
2695 bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
2751 bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
2808 SDValue StoredVal, SelectionDAG *CurDAG,
2811 SDValue &InputChain) {
2822 SDValue Load = StoredVal->getOperand(LoadOpNo);
2839 SmallVector<SDValue, 4> ChainOps;
2881 SDValue Chain = StoreNode->getChain();
2889 SDValue Op = Chain.getOperand(i);
2905 for (SDValue Op : StoredVal->ops())
2941 SDValue StoredVal = StoreNode->getOperand(1);
2973 SDValue InputChain;
2986 SDValue Base, Scale, Index, Disp, Segment;
3014 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
3031 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
3118 SDValue Operand = StoredVal->getOperand(1-LoadOpNo);
3149 SDValue CopyTo =
3153 const SDValue Ops[] = {Base, Scale, Index, Disp,
3158 const SDValue Ops[] = {Base, Scale, Index, Disp,
3202 SDValue NBits;
3234 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0));
3260 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0));
3302 SDValue M1 = Mask.getOperand(1);
3309 SDValue X;
3316 SDValue N0 = Node->getOperand(0);
3320 SDValue N1 = Node->getOperand(1);
3321 SDValue N01 = N0->getOperand(1);
3339 SDValue Mask = Node->getOperand(1);
3359 SDValue ImplDef = SDValue(
3363 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit, DL, MVT::i32);
3378 SDValue Extract = CurDAG->getNode(X86ISD::BZHI, DL, NVT, X, NBits);
3389 SDValue RealX = peekThroughOneUseTruncation(X);
3405 SDValue C8 = CurDAG->getConstant(8, DL, MVT::i8);
3406 SDValue Control = CurDAG->getNode(ISD::SHL, DL, MVT::i32, NBits, C8);
3412 SDValue ShiftAmt = X.getOperand(1);
3420 SDValue OrigShiftAmt = ShiftAmt;
3436 SDValue Extract = CurDAG->getNode(X86ISD::BEXTR, DL, XVT, X, Control);
3455 SDValue N0 = Node->getOperand(0);
3456 SDValue N1 = Node->getOperand(1);
3511 SDValue Control;
3544 SDValue Input = N0->getOperand(0);
3545 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3547 SDValue Ops[] = {
3561 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
3574 SDValue N0 = Node->getOperand(0);
3575 SDValue N1 = Node->getOperand(1);
3576 SDValue Imm = Node->getOperand(2);
3581 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3583 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
3594 SDValue Ops[] = { N0, N1, Imm };
3606 SDValue &InFlag) {
3607 SDValue N0 = Node->getOperand(0);
3608 SDValue N2 = Node->getOperand(2);
3609 SDValue Imm = Node->getOperand(4);
3614 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3616 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
3628 SDValue Ops[] = { N0, N2, Imm, InFlag };
3645 SDValue OrigShiftAmt = N->getOperand(1);
3646 SDValue ShiftAmt = OrigShiftAmt;
3656 SDValue NewShiftAmt;
3658 SDValue Add0 = ShiftAmt->getOperand(0);
3659 SDValue Add1 = ShiftAmt->getOperand(1);
3675 SDValue Zero = CurDAG->getConstant(0, DL, SubVT);
3676 SDValue Neg = CurDAG->getNode(ISD::SUB, DL, SubVT, Zero, Add1);
3729 SDValue Shift = N->getOperand(0);
3730 SDValue N1 = N->getOperand(1);
3817 SDValue X = Shift.getOperand(0);
3819 SDValue NewX = CurDAG->getNode(ISD::ANY_EXTEND, dl, NVT, X);
3824 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
3826 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst);
3828 SDValue NewSHL = CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp,
3848 SDValue X = Node->getOperand(0);
3849 SDValue OneVec = Node->getOperand(1);
3856 SDValue OneConstant, AllOnesVec;
3874 SDValue NewNode = CurDAG->getNode(NewOpcode, DL, VT, X, AllOnesVec);
3915 SDValue And0 = And->getOperand(0);
3944 SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT);
3945 SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask);
4110 bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
4111 SDValue InMask) {
4121 SDValue SetccOp0 = Setcc.getOperand(0);
4122 SDValue SetccOp1 = Setcc.getOperand(1);
4132 SDValue N0 = SetccOp0;
4138 SDValue Src0 = N0;
4139 SDValue Src1 = N0;
4143 SDValue N0Temp = N0;
4162 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Load;
4241 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4252 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4267 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4271 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4290 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4315 SDValue N0 = N->getOperand(0);
4316 SDValue N1 = N->getOperand(1);
4328 SDValue A = N1.getOperand(0);
4329 SDValue C = N1.getOperand(1);
4333 SDValue B;
4342 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
4343 SDValue Ternlog = CurDAG->getNode(X86ISD::VPTERNLOG, dl, NVT, A, B, C, Imm);
4393 SDValue Chain = CurDAG->getCopyToReg(Node->getOperand(0), dl, PtrReg,
4395 SDValue InFlag = Chain.getValue(1);
4427 const SDValue &Target = Node->getOperand(1);
4429 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, EVT(MVT::i64));
4430 SDValue Brind = CurDAG->getNode(ISD::BRIND, dl, MVT::Other,
4459 SDValue Blendv = CurDAG->getNode(
4481 SDValue N0 = Node->getOperand(0);
4482 SDValue N1 = Node->getOperand(1);
4530 SDValue N0 = Node->getOperand(0);
4531 SDValue N1 = Node->getOperand(1);
4602 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4604 SDValue Ops[] = { N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
4627 SDValue N0 = Node->getOperand(0);
4628 SDValue N1 = Node->getOperand(1);
4655 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4664 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
4677 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
4705 SDValue N0 = Node->getOperand(0);
4706 SDValue N1 = Node->getOperand(1);
4737 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4746 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg,
4749 SDValue Chain;
4751 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
4763 SDValue Ops[] = { N1, InFlag };
4772 SDValue ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
4782 SDValue ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
4796 SDValue N0 = Node->getOperand(0);
4797 SDValue N1 = Node->getOperand(1);
4842 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4846 SDValue InFlag;
4850 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
4853 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
4880 SDValue ClrNode = SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0);
4911 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
4933 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
4939 SDValue Result(RNode, 0);
4951 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
4960 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
4972 SDValue N0 = Node->getOperand(0);
4973 SDValue N1 = Node->getOperand(1);
4990 SDValue BEXTR = SDValue(NewNode, 0);
5019 SDValue Imm = CurDAG->getTargetConstant(TrailingZeros, dl, MVT::i64);
5020 SDValue Shift =
5030 SDValue Imm = CurDAG->getTargetConstant(LeadingZeros, dl, MVT::i64);
5031 SDValue Shift =
5090 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
5091 SDValue Reg = N0.getOperand(0);
5095 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5097 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
5151 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
5214 SDValue Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl, Node->getValueType(0),
5227 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
5228 std::vector<SDValue> &OutOps) {
5229 SDValue Op0, Op1, Op2, Op3, Op4;
lib/Target/X86/X86ISelLowering.cpp 1916 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
1916 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2230 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
2230 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
2408 static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
2408 static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
2422 SDValue ValToCopy = DAG.getBitcast(TempValLoc, ValArg);
2440 const SDLoc &Dl, SelectionDAG &DAG, SDValue &Arg,
2441 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, CCValAssign &VA,
2453 SDValue Lo, Hi;
2464 SDValue
2465 X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2468 const SmallVectorImpl<SDValue> &OutVals,
2486 SDValue Flag;
2487 SmallVector<SDValue, 6> RetOps;
2503 SDValue ValToCopy = OutVals[OutsIndex];
2568 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2627 SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,
2669 bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2673 SDValue TCChain = Chain;
2733 static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
2734 SDValue &Root, SelectionDAG &DAG,
2736 SDValue *InFlag = nullptr) {
2746 SDValue Lo, Hi;
2747 SDValue ArgValueLo, ArgValueHi;
2784 static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
2784 static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
2787 SDValue ValReturned = ValArg;
2820 SDValue X86TargetLowering::LowerCallResult(
2821 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2821 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2823 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
2870 SDValue Val;
2951 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
2951 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
2951 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
2952 SDValue Chain, ISD::ArgFlagsTy Flags,
2954 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
3009 SDValue
3010 X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
3057 SDValue PartAddr;
3084 SDValue Addr =
3105 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3106 SDValue Val = DAG.getLoad(
3173 SDValue X86TargetLowering::LowerFormalArguments(
3174 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3176 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3215 SDValue ArgValue;
3331 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[I]);
3369 SmallVector<SDValue, 6> LiveGPRs;
3370 SmallVector<SDValue, 8> LiveXMMRegs;
3371 SDValue ALVal;
3407 SmallVector<SDValue, 8> MemOps;
3408 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
3411 for (SDValue Val : LiveGPRs) {
3412 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
3414 SDValue Store =
3425 SmallVector<SDValue, 12> SaveXMMOps;
3476 SDValue RegVal = DAG.getCopyFromReg(Chain, dl, FR.VReg, FR.VT);
3537 SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
3537 SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
3537 SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
3538 SDValue Arg, const SDLoc &dl,
3543 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
3556 SDValue X86TargetLowering::EmitTailCallLoadRetAddr(
3557 SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
3557 SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
3570 static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
3571 SDValue Chain, SDValue RetAddrFrIdx,
3571 SDValue Chain, SDValue RetAddrFrIdx,
3580 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
3589 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
3589 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
3590 SDValue V2) {
3599 SDValue
3601 SmallVectorImpl<SDValue> &InVals) const {
3605 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
3607 SDValue Chain = CLI.Chain;
3608 SDValue Callee = CLI.Callee;
3735 SDValue RetAddrFrIdx;
3741 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3742 SmallVector<SDValue, 8> MemOpChains;
3743 SDValue StackPtr;
3763 SDValue Arg = OutVals[OutIndex];
3799 SDValue StackSlot =
3808 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
3907 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
3922 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
3924 SmallVector<SDValue, 8> MemOpChains2;
3925 SDValue FIN;
3944 SDValue Arg = OutVals[OutsIndex];
3957 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);
3986 SDValue InFlag;
4014 SmallVector<SDValue, 8> Ops;
4096 SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
4208 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
4221 const SDValue &TruncInput = Arg.getOperand(0);
4261 SDValue Ptr = Ld->getBasePtr();
4302 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
4305 const SmallVectorImpl<SDValue> &OutVals,
4430 SDValue Arg = OutVals[i];
4504 static bool MayFoldLoad(SDValue Op) {
4508 static bool MayFoldIntoStore(SDValue Op) {
4512 static bool MayFoldIntoZeroExtend(SDValue Op) {
4578 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
4686 bool isFP, SDValue &LHS, SDValue &RHS,
4686 bool isFP, SDValue &LHS, SDValue &RHS,
4866 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
4924 SDValue C) const {
4972 bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
5053 bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
5069 bool X86TargetLowering::hasAndNot(SDValue Y) const {
5086 bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
5086 bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
5092 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
5092 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
5131 bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
5354 bool X86::isZeroNode(SDValue Elt) {
5361 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
5364 SmallVector<SDValue, 32> Ops;
5378 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
5385 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
5391 static SDValue getConstVector(ArrayRef<APInt> Bits, APInt &Undefs,
5395 SmallVector<SDValue, 32> Ops;
5428 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
5433 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
5442 SDValue Vec;
5458 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
5458 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
5479 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
5489 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
5489 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
5497 static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
5497 static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
5503 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5503 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5503 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5523 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
5533 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5533 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5533 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5541 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
5541 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
5547 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
5555 static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
5555 static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
5570 static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops) {
5580 SDValue Src = N->getOperand(0);
5581 SDValue Sub = N->getOperand(1);
5609 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
5610 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
5635 SmallVector<SDValue, 4> Subs;
5637 SmallVector<SDValue, 2> SubOps;
5638 for (SDValue Op : Ops) {
5650 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
5650 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
5654 SDValue Vec = Op.getOperand(0);
5655 SDValue SubVec = Op.getOperand(1);
5656 SDValue Idx = Op.getOperand(2);
5672 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
5696 SDValue Undef = DAG.getUNDEF(WideOpVT);
5700 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
5752 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
5777 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
5784 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
5797 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
5797 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
5797 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
5805 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
5812 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
5818 SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts));
5838 static SDValue getExtendInVec(unsigned Opcode, const SDLoc &DL, EVT VT,
5839 SDValue In, SelectionDAG &DAG) {
5866 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
5866 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
5873 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
5879 SmallVector<SDValue, 2> CatOps;
5881 for (SDValue &CatOp : CatOps) {
5882 SDValue NotCat = IsNOT(CatOp, DAG);
5892 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
5893 SDValue V1, SDValue V2) {
5893 SDValue V1, SDValue V2) {
5900 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
5901 SDValue V1, SDValue V2) {
5901 SDValue V1, SDValue V2) {
5911 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
5911 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
5916 SDValue V1 = IsZero
5930 SDValue Ptr = Load->getBasePtr();
5942 static const Constant *getTargetConstantFromNode(SDValue Op) {
5954 static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
6072 const SDValue &Src = Op.getOperand(i);
6089 const SDValue &Src = Op.getOperand(i);
6145 SDValue Ptr = MemIntr->getBasePtr();
6292 bool isConstantSplat(SDValue Op, APInt &SplatVal) {
6318 static bool getTargetShuffleMaskIndices(SDValue MaskNode,
6415 SmallVectorImpl<SDValue> &Ops,
6421 SDValue ImmN;
6551 SDValue N0 = N->getOperand(0);
6572 SDValue MaskNode = N->getOperand(1);
6585 SDValue MaskNode = N->getOperand(1);
6639 SDValue MaskNode = N->getOperand(2);
6640 SDValue CtrlNode = N->getOperand(3);
6656 SDValue MaskNode = N->getOperand(2);
6668 SDValue MaskNode = N->getOperand(0);
6683 SDValue MaskNode = N->getOperand(1);
6725 static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
6726 SmallVectorImpl<SDValue> &Ops,
6737 SDValue V1 = Ops[0];
6738 SDValue V2 = IsUnary ? V1 : Ops[1];
6772 SDValue V = M < Size ? V1 : V2;
6844 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
6844 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
6852 static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
6854 SmallVectorImpl<SDValue> &Ops,
6886 SDValue N0 = N.getOperand(0);
6887 SDValue N1 = N.getOperand(1);
6944 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
6945 SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
6949 SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
6976 SDValue Src = N.getOperand(0);
6977 SDValue Sub = N.getOperand(1);
6999 SmallVector<SDValue, 2> SubInputs;
7019 for (SDValue &SubInput : SubInputs) {
7042 SDValue N0 = N.getOperand(0);
7043 SDValue SrcExtract;
7057 SDValue SrcVec = SrcExtract.getOperand(0);
7074 SDValue InVec = N.getOperand(0);
7075 SDValue InScl = N.getOperand(1);
7076 SDValue InIndex = N.getOperand(2);
7097 SDValue ExVec = InScl.getOperand(0);
7098 SDValue ExIndex = InScl.getOperand(1);
7112 SDValue N0 = N.getOperand(0);
7113 SDValue N1 = N.getOperand(1);
7180 SDValue Src = N.getOperand(0);
7203 SDValue Src = N.getOperand(0);
7236 static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
7239 SmallVector<SDValue, 16> UsedInputs;
7280 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
7281 SmallVectorImpl<SDValue> &Inputs,
7303 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
7303 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
7320 static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
7325 SDValue V = SDValue(N, 0);
7337 SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0)
7348 SmallVector<SDValue, 16> ShuffleOps;
7362 SDValue NewV = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
7370 SDValue Vec = N->getOperand(0);
7371 SDValue Sub = N->getOperand(1);
7384 SDValue Src = N->getOperand(0);
7410 static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros,
7410 static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros,
7421 SDValue V;
7452 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
7452 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
7465 SDValue V;
7475 SDValue Elt;
7484 SDValue NextElt = Op.getOperand(i + 1);
7517 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
7517 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
7530 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
7530 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
7545 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
7547 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
7548 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
7555 SDValue Elt = Op.getOperand(i);
7564 SDValue FirstNonZero;
7569 SDValue Elt = Op.getOperand(i);
7584 SDValue V1 = FirstNonZero.getOperand(0);
7588 SDValue Elt;
7608 SDValue VZeroOrUndef = (Zeroable == Undefs)
7620 SDValue V2 = Elt.getOperand(0);
7629 SDValue Current = Op->getOperand(i);
7630 SDValue SrcVector = Current->getOperand(0);
7651 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
7657 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
7657 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
7665 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
7669 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
7669 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
7676 SDValue Ptr = LD->getBasePtr();
7700 SDValue Chain = LD->getChain();
7731 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
7743 static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
7769 SDValue Src = Elt.getOperand(0);
7790 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
7790 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
7810 SDValue Elt = peekThroughBitcasts(Elts[i]);
7852 SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
7902 SDValue NewLd =
7952 SDValue V = CreateLoad(VT, LDBase);
7953 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
7966 SDValue HalfLD =
7984 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
7985 SDValue ResNode =
8008 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
8012 SDValue Elt = peekThroughBitcasts(Elts[i]);
8036 if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
8040 SDValue Broadcast = DAG.getNode(Opcode, DL, BroadcastVT, RepeatLoad);
8053 static SDValue combineToConsecutiveLoads(EVT VT, SDNode *N, const SDLoc &DL,
8057 SmallVector<SDValue, 64> Elts;
8059 if (SDValue Elt = getShuffleScalarElt(N, i, DAG, 0)) {
8116 static SDValue isSplatZeroExtended(const BuildVectorSDNode *Op,
8118 SDValue ExtValue = Op->getOperand(0);
8155 static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
8171 SDValue Ld = BVOp->getSplatValue(&UndefElements);
8182 SDValue BOperand;
8183 SDValue ZeroExtended = isSplatZeroExtended(BVOp, NumElts, EltType);
8194 SDValue Brdcst =
8228 SDValue CP = DAG.getConstantPool(C, PVT);
8236 SDValue Brdcst = DAG.getNode(X86ISD::VBROADCAST, dl,
8252 SDValue CP = DAG.getConstantPool(C, PVT);
8260 SDValue Brdcst = DAG.getNode(X86ISD::VBROADCAST, dl,
8268 SDValue VCP = DAG.getConstantPool(VecC, PVT);
8275 SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);
8336 SDValue CP =
8379 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
8380 SDValue ExtIdx) {
8396 SDValue ShuffleVec = SVOp->getOperand(0);
8409 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
8409 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
8420 SDValue VecIn1;
8421 SDValue VecIn2;
8440 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
8441 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
8472 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
8481 static SDValue ConvertI1VectorToInteger(SDValue Op, SelectionDAG &DAG) {
8481 static SDValue ConvertI1VectorToInteger(SDValue Op, SelectionDAG &DAG) {
8487 SDValue In = Op.getOperand(idx);
8496 static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
8496 static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
8514 SDValue In = Op.getOperand(idx);
8535 SDValue Cond = Op.getOperand(SplatIdx);
8546 SDValue DstVec;
8549 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
8550 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
8556 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
8597 SDValue &V0, SDValue &V1) {
8597 SDValue &V0, SDValue &V1) {
8613 SDValue Op = N->getOperand(i + BaseIdx);
8629 SDValue Op0 = Op.getOperand(0);
8630 SDValue Op1 = Op.getOperand(1);
8661 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
8708 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
8708 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
8708 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
8717 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
8718 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
8719 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
8720 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
8723 SDValue LO = DAG.getUNDEF(NewVT);
8724 SDValue HI = DAG.getUNDEF(NewVT);
8751 SDValue &Opnd0, SDValue &Opnd1,
8751 SDValue &Opnd0, SDValue &Opnd1,
8760 SDValue InVec0 = DAG.getUNDEF(VT);
8761 SDValue InVec1 = DAG.getUNDEF(VT);
8771 SDValue Op = BV->getOperand(i);
8782 SDValue Op0 = Op.getOperand(0);
8783 SDValue Op1 = Op.getOperand(1);
8879 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8879 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8879 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8905 static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
8908 SDValue Opnd0, Opnd1;
8919 SDValue Opnd2;
8941 unsigned &HOpcode, SDValue &V0, SDValue &V1) {
8941 unsigned &HOpcode, SDValue &V0, SDValue &V1) {
8959 SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
8979 SDValue Op0 = Op.getOperand(0);
8980 SDValue Op1 = Op.getOperand(1);
8998 SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
9027 static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
9029 SDValue V0, SDValue V1) {
9029 SDValue V0, SDValue V1) {
9057 SDValue Half = DAG.getNode(HOpcode, SDLoc(BV), HalfVT, V0, V1);
9065 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
9083 SDValue V0, V1;
9106 SDValue InVec0, InVec1;
9108 SDValue InVec2, InVec3;
9137 SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
9138 SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
9186 static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
9222 SmallVector<SDValue, 4> LHSElts, RHSElts;
9223 for (SDValue Elt : Op->ops()) {
9224 SDValue LHS = Elt.getOperand(0);
9225 SDValue RHS = Elt.getOperand(1);
9248 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
9249 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
9256 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
9256 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
9281 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
9281 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
9281 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
9387 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
9388 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
9389 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
9390 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
9396 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
9397 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
9398 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
9399 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
9405 SDValue Idx = Ops[2];
9412 SDValue Ops[] = {LoLo, HiHi, IndicesVec};
9435 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9437 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9445 SDValue Res = DAG.getSelectCC(
9462 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
9469 SDValue LoLo =
9471 SDValue HiHi =
9481 SDValue Res = DAG.getSelectCC(
9520 SDValue Res = Opcode == X86ISD::VPERMV
9537 static SDValue
9538 LowerBUILD_VECTORAsVariablePermute(SDValue V, SelectionDAG &DAG,
9540 SDValue SrcVec, IndicesVec;
9545 SDValue Op = V.getOperand(Idx);
9556 SDValue ExtractedIndex = Op->getOperand(1);
9582 SDValue
9583 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
9594 if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
9598 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG))
9600 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
9602 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
9604 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, DAG))
9613 SmallSet<SDValue, 8> Values;
9616 SDValue Elt = Op.getOperand(i);
9652 SDValue VarElt;
9653 SDValue InsIndex;
9655 SDValue Elt = Op.getOperand(i);
9668 SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
9676 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
9679 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
9693 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
9700 SDValue Item = Op.getOperand(Idx);
9764 SDValue Item = Op.getOperand(Idx);
9776 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, DAG, Subtarget))
9781 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
9782 if (SDValue LD =
9791 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
9804 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
9819 SDValue Lower =
9821 SDValue Upper = DAG.getBuildVector(
9833 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
9842 if (SDValue V = LowerBuildVectorv16i8(Op, NonZeros, NumNonZero, NumZero,
9847 if (SDValue V = LowerBuildVectorv8i16(Op, NonZeros, NumNonZero, NumZero,
9853 if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget))
9858 SmallVector<SDValue, 8> Ops(NumElems);
9899 if (SDValue Sh = buildFromShuffleMostly(Op, DAG))
9904 SDValue Result;
9921 SmallVector<SDValue, 8> Ops(NumElems);
9951 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9951 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9964 SDValue SubVec = Op.getOperand(i);
9980 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9982 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9988 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
10009 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
10009 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
10022 SDValue SubVec = Op.getOperand(i);
10043 SDValue SubVec = Op.getOperand(Idx);
10056 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
10060 SDValue SubVec = Op.getOperand(Idx);
10069 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10071 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10081 SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT,
10088 static SDValue LowerCONCAT_VECTORS(SDValue Op,
10088 static SDValue LowerCONCAT_VECTORS(SDValue Op,
10252 static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
10252 static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
10290 SDValue V1 = SDValue(),
10291 SDValue V2 = SDValue()) {
10328 SDValue Cond) {
10336 SDValue CondElt = Cond.getOperand(i);
10422 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
10436 SDValue V1, SDValue V2) {
10436 SDValue V1, SDValue V2) {
10457 SDValue V = M < Size ? V1 : V2;
10468 SDValue Op = V.getOperand(M / Scale);
10493 SDValue Op = V.getOperand((M * Scale) + j);
10538 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
10539 ArrayRef<int> Mask, SDValue V1,
10540 SDValue V2, const APInt &Zeroable,
10552 SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
10554 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
10556 SDValue V;
10569 SDValue SrcV = (M >= Size ? V2 : V1);
10591 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
10591 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
10596 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
10598 ArrayRef<int> Mask, SDValue &V1,
10599 SDValue &V2, SelectionDAG &DAG,
10608 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
10612 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
10614 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
10615 SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
10619 static bool matchVectorShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
10619 static bool matchVectorShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
10706 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
10707 ArrayRef<int> Mask, SDValue V1, SDValue V2,
10707 ArrayRef<int> Mask, SDValue V1, SDValue V2,
10770 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
10771 MVT VT, SDValue V1, SDValue V2,
10771 MVT VT, SDValue V1, SDValue V2,
10801 SDValue Src = V1.getOperand(0).getOperand(0);
10826 static bool matchVectorShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1,
10827 SDValue &V2, unsigned &PackOpcode,
10837 SDValue VV1 = DAG.getBitcast(PackVT, N1);
10838 SDValue VV2 = DAG.getBitcast(PackVT, N2);
10878 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
10879 SDValue V1, SDValue V2, SelectionDAG &DAG,
10879 SDValue V1, SDValue V2, SelectionDAG &DAG,
10895 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
10895 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
10896 SDValue V2, ArrayRef<int> Mask,
10902 SDValue Zero, AllOnes;
10921 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
10922 SDValue V;
10938 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
10941 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
10950 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
10950 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
10951 SDValue V2, ArrayRef<int> Mask,
10955 SDValue Zero = DAG.getConstant(0, DL, EltVT);
10956 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10957 SmallVector<SDValue, 16> MaskOps;
10964 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
10970 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10970 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10970 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10971 SDValue PreservedSrc,
10975 static bool matchVectorShuffleAsBlend(SDValue V1, SDValue V2,
10975 static bool matchVectorShuffleAsBlend(SDValue V1, SDValue V2,
11033 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
11033 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
11034 SDValue V2, ArrayRef<int> Original,
11087 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11089 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11104 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11111 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11142 SmallVector<SDValue, 32> VSELECTMask;
11166 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11175 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11188 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
11189 SDValue V1, SDValue V2,
11189 SDValue V1, SDValue V2,
11218 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11227 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
11228 SDValue V1, SDValue V2,
11228 SDValue V1, SDValue V2,
11237 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
11246 SDValue &Op = Ops[Elt & 1];
11284 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
11290 static SDValue lowerShuffleAsByteRotateAndPermute(
11291 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11291 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11347 SDValue Rotate = DAG.getBitcast(
11381 static SDValue lowerShuffleAsDecomposedShuffleBlend(
11382 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11382 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11405 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11408 if (SDValue UnpackPerm = lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask,
11411 if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
11415 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11428 static int matchShuffleAsRotate(SDValue &V1, SDValue &V2, ArrayRef<int> Mask) {
11428 static int matchShuffleAsRotate(SDValue &V1, SDValue &V2, ArrayRef<int> Mask) {
11439 SDValue Lo, Hi;
11465 SDValue MaskV = M < NumElts ? V1 : V2;
11470 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
11512 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11512 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11534 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11534 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11535 SDValue V2, ArrayRef<int> Mask,
11540 SDValue Lo = V1, Hi = V2;
11571 SDValue LoShift =
11574 SDValue HiShift =
11591 static SDValue lowerShuffleAsRotate(const SDLoc &DL, MVT VT, SDValue V1,
11591 static SDValue lowerShuffleAsRotate(const SDLoc &DL, MVT VT, SDValue V1,
11592 SDValue V2, ArrayRef<int> Mask,
11602 SDValue Lo = V1, Hi = V2;
11612 static SDValue lowerVectorShuffleAsByteShiftMask(
11613 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11613 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11636 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11759 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11759 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11760 SDValue V2, ArrayRef<int> Mask,
11768 SDValue V = V1;
11795 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
11795 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
11816 SDValue Src;
11822 SDValue &V = (M < Size ? V1 : V2);
11851 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
11851 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
11863 SDValue Base;
11879 SDValue Insert;
11917 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
11917 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
11918 SDValue V2, ArrayRef<int> Mask,
11944 static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
11945 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
12025 SDValue Lo = DAG.getBitcast(
12034 SDValue Hi = DAG.getBitcast(
12047 SDValue PSHUFBMask[16];
12083 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
12106 static SDValue lowerShuffleAsZeroOrAnyExtend(
12107 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12107 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12121 SDValue InputV;
12141 SDValue V = M < NumElements ? V1 : V2;
12191 if (SDValue V = Lower(NumElements / NumExtElements))
12212 if (SDValue V = CanZExtLowHalf()) {
12225 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12225 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12241 SDValue S = V.getOperand(Idx);
12253 static bool isShuffleFoldableLoad(SDValue V) {
12262 static SDValue lowerShuffleAsElementInsertion(
12263 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12263 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12284 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
12360 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12360 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12395 SDValue Scalar = V0.getOperand(V0BroadcastIdx);
12434 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12434 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12435 SDValue N1, ArrayRef<int> Mask,
12449 SDValue WideVec = N0.getOperand(0);
12477 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12489 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12489 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12490 SDValue V2, ArrayRef<int> Mask,
12526 SDValue V = V1;
12541 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
12574 if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
12601 SDValue BaseAddr = Ld->getOperand(1);
12605 SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
12680 static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12680 static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12768 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12768 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12768 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12793 static SDValue lowerShuffleAsPermuteAndUnpack(
12794 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12794 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12861 if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
12908 static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12909 const APInt &Zeroable, SDValue V1, SDValue V2,
12909 const APInt &Zeroable, SDValue V1, SDValue V2,
12918 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
12945 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12950 if (SDValue Insertion = lowerShuffleAsElementInsertion(
12957 if (SDValue Insertion = lowerShuffleAsElementInsertion(
12965 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
12973 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
12978 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
12992 static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12993 const APInt &Zeroable, SDValue V1, SDValue V2,
12993 const APInt &Zeroable, SDValue V1, SDValue V2,
13002 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
13024 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13028 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
13034 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13040 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13048 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
13053 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
13060 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v2i64, V1, V2, Mask,
13064 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
13090 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
13091 ArrayRef<int> Mask, SDValue V1,
13092 SDValue V2, SelectionDAG &DAG) {
13093 SDValue LowV = V1, HighV = V2;
13177 static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13178 const APInt &Zeroable, SDValue V1, SDValue V2,
13178 const APInt &Zeroable, SDValue V1, SDValue V2,
13189 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
13224 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13233 if (SDValue V = lowerShuffleAsElementInsertion(
13238 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
13243 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13247 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13262 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13273 static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13274 const APInt &Zeroable, SDValue V1, SDValue V2,
13274 const APInt &Zeroable, SDValue V1, SDValue V2,
13284 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
13293 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
13315 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13319 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
13325 if (SDValue V = lowerShuffleAsElementInsertion(
13333 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
13337 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
13342 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
13349 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v4i32, V1, V2, Mask,
13353 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
13369 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
13379 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13380 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13381 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13401 static SDValue lowerV8I16GeneralSingleInputShuffle(
13402 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
13897 static SDValue lowerShuffleAsBlendOfPSHUFBs(
13898 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13898 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13907 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13908 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13938 SDValue V;
13960 static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13961 const APInt &Zeroable, SDValue V1, SDValue V2,
13961 const APInt &Zeroable, SDValue V1, SDValue V2,
13970 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
13978 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,
13983 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
13988 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13992 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13997 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
14012 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
14018 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
14024 if (SDValue V = lowerShuffleAsElementInsertion(
14032 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
14036 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
14041 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14045 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14050 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
14054 if (SDValue BitBlend =
14059 if (SDValue V = lowerVectorShuffleAsByteShiftMask(
14064 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
14148 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
14149 ArrayRef<int> Mask, SDValue V1,
14150 SDValue V2, SelectionDAG &DAG) {
14154 SDValue MaskNode = getConstVector(Mask, MaskVecVT, DAG, DL, true);
14168 static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14169 const APInt &Zeroable, SDValue V1, SDValue V2,
14169 const APInt &Zeroable, SDValue V1, SDValue V2,
14177 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
14182 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14187 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14192 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14198 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14207 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14211 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14308 if (SDValue V = tryToWidenViaDuplication())
14312 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
14317 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14321 if (SDValue V = lowerVectorShuffleAsByteShiftMask(
14342 SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
14350 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
14362 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
14372 if (SDValue V = lowerShuffleAsByteRotateAndPermute(
14382 if (SDValue V = lowerShuffleAsElementInsertion(
14386 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14405 SmallVector<SDValue, 16> ByteClearOps(16, DAG.getConstant(0, DL, MVT::i8));
14408 SDValue ByteClearMask = DAG.getBuildVector(MVT::v16i8, DL, ByteClearOps);
14416 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
14433 SDValue V = V1;
14441 SDValue VLoHalf, VHiHalf;
14465 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14473 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14474 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14483 static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
14484 MVT VT, SDValue V1, SDValue V2,
14484 MVT VT, SDValue V1, SDValue V2,
14512 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14512 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14513 SDValue V2, ArrayRef<int> Mask,
14539 SDValue LoV, HiV;
14549 SmallVector<SDValue, 16> LoOps, HiOps;
14561 SDValue LoV1, HiV1, LoV2, HiV2;
14602 SDValue V1Blend, V2Blend;
14625 SDValue Lo = HalfBlend(LoMask);
14626 SDValue Hi = HalfBlend(HiMask);
14638 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14638 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14639 SDValue V2, ArrayRef<int> Mask,
14697 static SDValue lowerShuffleAsLanePermuteAndPermute(
14698 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14698 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14748 SDValue LanePermute = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask);
14759 static SDValue lowerShuffleAsLanePermuteAndShuffle(
14760 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14760 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14803 SDValue Flipped = DAG.getBitcast(PVT, V1);
14811 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
14811 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
14812 SDValue V2, ArrayRef<int> Mask,
14830 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
14842 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
14858 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
14914 static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
14915 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14915 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15041 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15058 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15134 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15134 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15134 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15148 SDValue V = (HalfIdx < 2 ? V1 : V2);
15155 SDValue Half1 = getHalfVector(HalfIdx1);
15156 SDValue Half2 = getHalfVector(HalfIdx2);
15157 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15159 SDValue Op0 = V;
15160 SDValue Op1 = DAG.getUNDEF(HalfVT);
15174 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15174 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15175 SDValue V2, ArrayRef<int> Mask,
15194 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15204 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15300 static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
15301 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15301 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15339 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15450 SDValue RepeatedShuffle = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
15466 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15466 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15511 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15511 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15512 SDValue V2, ArrayRef<int> Mask,
15538 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
15539 SDValue V1, SDValue V2,
15539 SDValue V1, SDValue V2,
15561 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
15575 static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15576 const APInt &Zeroable, SDValue V1, SDValue V2,
15576 const APInt &Zeroable, SDValue V1, SDValue V2,
15583 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
15589 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
15613 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15618 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
15628 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
15631 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
15636 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
15648 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15658 if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
15664 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
15683 static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15684 const APInt &Zeroable, SDValue V1, SDValue V2,
15684 const APInt &Zeroable, SDValue V1, SDValue V2,
15692 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15696 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
15701 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
15726 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
15732 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v4i64, V1, V2, Mask,
15736 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
15742 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
15747 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
15758 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15768 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
15781 static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15782 const APInt &Zeroable, SDValue V1, SDValue V2,
15782 const APInt &Zeroable, SDValue V1, SDValue V2,
15789 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
15794 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
15816 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
15826 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15833 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
15847 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
15853 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
15861 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
15880 static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15881 const APInt &Zeroable, SDValue V1, SDValue V2,
15881 const APInt &Zeroable, SDValue V1, SDValue V2,
15892 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
15901 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask,
15905 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
15910 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
15927 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
15932 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
15938 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v8i32, V1, V2, Mask,
15942 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
15948 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
15954 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15961 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
15969 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
15970 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
15971 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
15978 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
15991 static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15992 const APInt &Zeroable, SDValue V1, SDValue V2,
15992 const APInt &Zeroable, SDValue V1, SDValue V2,
16003 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16008 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16012 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16017 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16021 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16026 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
16031 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16037 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16045 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16063 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16073 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16078 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16091 static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16092 const APInt &Zeroable, SDValue V1, SDValue V2,
16092 const APInt &Zeroable, SDValue V1, SDValue V2,
16103 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16108 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16112 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16117 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
16121 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
16126 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
16131 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16137 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16144 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16152 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16162 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16167 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16175 if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16189 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16190 SDValue V1, SDValue V2, const APInt &Zeroable,
16190 SDValue V1, SDValue V2, const APInt &Zeroable,
16199 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16204 if (SDValue V =
16219 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16222 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16254 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
16255 const APInt &Zeroable, SDValue V1, SDValue V2,
16255 const APInt &Zeroable, SDValue V1, SDValue V2,
16275 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16289 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
16323 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
16329 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
16337 SDValue Op = WidenedMask[i] >= 4 ? V2 : V1;
16354 static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16355 const APInt &Zeroable, SDValue V1, SDValue V2,
16355 const APInt &Zeroable, SDValue V1, SDValue V2,
16384 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
16388 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
16392 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
16396 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
16400 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
16408 static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16409 const APInt &Zeroable, SDValue V1, SDValue V2,
16409 const APInt &Zeroable, SDValue V1, SDValue V2,
16433 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
16436 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16448 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
16453 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
16461 static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16462 const APInt &Zeroable, SDValue V1, SDValue V2,
16462 const APInt &Zeroable, SDValue V1, SDValue V2,
16490 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
16495 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
16500 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v8i64, V1, V2, Mask,
16505 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
16509 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
16512 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
16516 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
16524 static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16525 const APInt &Zeroable, SDValue V1, SDValue V2,
16525 const APInt &Zeroable, SDValue V1, SDValue V2,
16535 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16552 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
16557 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
16562 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v16i32, V1, V2, Mask,
16568 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
16575 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
16576 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
16577 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
16582 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
16586 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
16593 static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16594 const APInt &Zeroable, SDValue V1, SDValue V2,
16594 const APInt &Zeroable, SDValue V1, SDValue V2,
16605 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16610 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
16614 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
16619 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
16634 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
16638 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
16646 static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16647 const APInt &Zeroable, SDValue V1, SDValue V2,
16647 const APInt &Zeroable, SDValue V1, SDValue V2,
16658 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16663 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
16667 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
16672 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
16677 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
16681 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
16691 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16695 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
16702 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16715 static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
16716 MVT VT, SDValue V1, SDValue V2,
16716 MVT VT, SDValue V1, SDValue V2,
16729 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16734 if (SDValue V =
16739 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
16766 static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
16767 MVT VT, SDValue V1, SDValue V2,
16767 MVT VT, SDValue V1, SDValue V2,
16800 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
16846 static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
16847 MVT VT, SDValue V1, SDValue V2,
16847 MVT VT, SDValue V1, SDValue V2,
16881 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
16890 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
16896 for (SDValue V : { V1, V2 }) {
16903 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
16960 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17042 static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget,
17042 static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget,
17046 SDValue V1 = Op.getOperand(0);
17047 SDValue V2 = Op.getOperand(1);
17116 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17158 if (SDValue V = lowerShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))
17178 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17178 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17181 SDValue Cond = Op.getOperand(0);
17182 SDValue LHS = Op.getOperand(1);
17183 SDValue RHS = Op.getOperand(2);
17195 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
17195 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
17196 SDValue Cond = Op.getOperand(0);
17197 SDValue LHS = Op.getOperand(1);
17198 SDValue RHS = Op.getOperand(2);
17209 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
17234 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
17275 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
17281 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
17281 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
17289 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
17308 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17325 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
17325 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
17327 SDValue Vec = Op.getOperand(0);
17330 SDValue Idx = Op.getOperand(1);
17344 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
17345 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
17371 SDValue
17372 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
17375 SDValue Vec = Op.getOperand(0);
17377 SDValue Idx = Op.getOperand(1);
17449 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
17455 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
17465 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17476 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
17518 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
17518 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
17521 SDValue Vec = Op.getOperand(0);
17522 SDValue Elt = Op.getOperand(1);
17523 SDValue Idx = Op.getOperand(2);
17532 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
17539 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
17545 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
17545 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
17555 SDValue N0 = Op.getOperand(0);
17556 SDValue N1 = Op.getOperand(1);
17557 SDValue N2 = Op.getOperand(2);
17575 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
17591 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
17598 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
17680 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
17680 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
17710 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
17718 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
17718 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
17725 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
17725 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
17731 SDValue Vec = Op.getOperand(0);
17732 SDValue Idx = Op.getOperand(1);
17786 SDValue
17787 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
17795 SDValue Result = DAG.getTargetConstantPool(
17809 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
17809 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
17817 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
17830 SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
17830 SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
17835 SDValue
17836 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
17844 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
17858 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
17858 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
17885 SDValue Result;
17929 SDValue
17930 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
17934 static SDValue
17935 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
17936 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
17941 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
17950 SDValue Ops[] = { Chain, TGA, *InFlag };
17953 SDValue Ops[] = { Chain, TGA };
17961 SDValue Flag = Chain.getValue(1);
17966 static SDValue
17969 SDValue InFlag;
17971 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
17980 static SDValue
17987 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
17998 SDValue Base;
18003 SDValue InFlag;
18004 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18017 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18020 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18027 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18036 SDValue ThreadPointer =
18060 SDValue TGA =
18063 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18081 SDValue
18082 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
18125 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18128 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18138 SDValue Chain = DAG.getEntryNode();
18141 SDValue Args[] = { Chain, Offset };
18170 SDValue Chain = DAG.getEntryNode();
18181 SDValue TlsArray = Subtarget.is64Bit()
18187 SDValue ThreadPointer =
18190 SDValue res;
18195 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
18203 SDValue Scale =
18213 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18216 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
18229 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
18229 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
18235 SDValue ShOpLo = Op.getOperand(0);
18236 SDValue ShOpHi = Op.getOperand(1);
18237 SDValue ShAmt = Op.getOperand(2);
18241 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
18243 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
18247 SDValue Tmp2, Tmp3;
18259 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
18261 SDValue Cond = DAG.getSetCC(dl, MVT::i8, AndNode,
18264 SDValue Hi, Lo;
18276 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
18276 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
18283 SDValue Op0 = Op.getOperand(0);
18284 SDValue Op1 = Op.getOperand(1);
18285 SDValue Amt = Op.getOperand(2);
18328 static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
18328 static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
18332 SDValue Src = Op.getOperand(0);
18348 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
18349 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
18379 static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
18379 static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
18383 SDValue Extract = Cast.getOperand(0);
18390 SDValue VecOp = Extract.getOperand(0);
18413 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
18418 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
18418 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
18420 SDValue Src = Op.getOperand(0);
18428 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
18450 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
18453 SDValue ValueToStore = Op.getOperand(0);
18465 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
18466 SDValue Chain = DAG.getStore(
18472 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
18472 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
18472 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
18473 SDValue StackSlot,
18497 SDValue FILDOps[] = {Chain, StackSlot};
18498 SDValue Result =
18504 SDValue InFlag = Result.getValue(2);
18513 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
18515 SDValue FSTOps[] = {Chain, Result, StackSlot, InFlag};
18541 static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG,
18541 static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG,
18563 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, 16);
18573 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, 16);
18576 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
18578 SDValue CLod0 =
18582 SDValue Unpck1 =
18585 SDValue CLod1 =
18589 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
18591 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
18592 SDValue Result;
18597 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
18606 static SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG,
18606 static SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG,
18610 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl,
18614 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
18625 SDValue Or = DAG.getNode(
18637 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
18643 static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,
18643 static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,
18649 SDValue N0 = Op.getOperand(0);
18661 SDValue HalfWord = DAG.getConstant(16, DL, MVT::v4i32);
18662 SDValue HalfWordMask = DAG.getConstant(0x0000FFFF, DL, MVT::v4i32);
18665 SDValue TWOHW = DAG.getConstantFP((double)(1 << 16), DL, MVT::v2f64);
18668 SDValue HI = DAG.getNode(ISD::SRL, DL, MVT::v4i32, N0, HalfWord);
18669 SDValue LO = DAG.getNode(ISD::AND, DL, MVT::v4i32, N0, HalfWordMask);
18671 SDValue fHI = DAG.getNode(X86ISD::CVTSI2P, DL, MVT::v2f64, HI);
18673 SDValue fLO = DAG.getNode(X86ISD::CVTSI2P, DL, MVT::v2f64, LO);
18679 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
18679 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
18703 SDValue V = Op->getOperand(0);
18723 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
18725 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
18728 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
18729 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
18731 SDValue Low, High;
18735 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
18736 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
18743 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
18744 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
18750 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
18752 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
18760 SDValue VecCstFAdd = DAG.getConstantFP(
18764 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
18766 SDValue FHigh =
18769 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
18773 static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
18773 static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
18775 SDValue N0 = Op.getOperand(0);
18791 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
18791 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
18793 SDValue N0 = Op.getOperand(0);
18805 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
18821 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
18832 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
18834 SDValue OffsetSlot = DAG.getMemBasePlusOffset(StackSlot, 4, dl);
18835 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
18837 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
18839 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
18844 SDValue ValueToStore = Op.getOperand(0);
18850 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, ValueToStore, StackSlot,
18863 SDValue Ops[] = { Store, StackSlot };
18864 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
18870 SDValue SignSet = DAG.getSetCC(
18875 SDValue FudgePtr = DAG.getConstantPool(
18879 SDValue Zero = DAG.getIntPtrConstant(0, dl);
18880 SDValue Four = DAG.getIntPtrConstant(4, dl);
18881 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Zero, Four);
18886 SDValue Fudge = DAG.getExtLoad(
18892 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
18903 SDValue
18904 X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
18939 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
18941 SDValue Chain = DAG.getEntryNode();
18942 SDValue Value = Op.getOperand(0);
18943 SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
18976 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
18978 SDValue Cmp = DAG.getSetCC(DL,
18986 SDValue Sub = DAG.getNode(ISD::FSUB, DL, TheVT, Value, ThreshVal);
19001 SDValue Ops[] = { Chain, StackSlot };
19014 SDValue Ops[] = { Chain, Value, StackSlot };
19015 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
19019 SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI);
19028 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
19028 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
19031 SDValue In = Op.getOperand(0);
19078 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
19086 SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
19087 SDValue Undef = DAG.getUNDEF(InVT);
19089 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
19096 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
19096 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
19099 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
19101 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
19105 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
19109 static SDValue LowerZERO_EXTEND_Mask(SDValue Op,
19109 static SDValue LowerZERO_EXTEND_Mask(SDValue Op,
19113 SDValue In = Op->getOperand(0);
19122 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
19148 SDValue One = DAG.getConstant(1, DL, WideVT);
19149 SDValue Zero = DAG.getConstant(0, DL, WideVT);
19151 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
19167 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
19167 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
19169 SDValue In = Op.getOperand(0);
19184 static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
19184 static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
19232 SDValue Res = DAG.getNode(Opcode, DL, OutVT, In, In);
19239 SDValue Lo = extractSubVector(In, 0 * NumSubElts, DAG, DL, SrcSizeInBits / 2);
19240 SDValue Hi = extractSubVector(In, 1 * NumSubElts, DAG, DL, SrcSizeInBits / 2);
19250 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
19259 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
19285 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
19289 static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
19289 static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
19294 SDValue In = Op.getOperand(0);
19336 SDValue Lo = extract128BitVector(In, 0, DAG, DL);
19337 SDValue Hi = extract128BitVector(In, 8, DAG, DL);
19365 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
19365 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
19368 SDValue In = Op.getOperand(0);
19384 SDValue Lo, Hi;
19421 if (SDValue V =
19428 if (SDValue V =
19445 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
19447 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
19475 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
19478 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
19496 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
19504 SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
19506 SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
19514 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
19514 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
19517 SDValue Src = Op.getOperand(0);
19546 SDValue Res = DAG.getNode(Opc, dl, ResVT, Src);
19579 SDValue Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);
19592 SDValue Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
19601 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned))
19607 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
19607 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
19610 SDValue In = Op.getOperand(0);
19625 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
19625 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
19627 SDValue In = Op.getOperand(0);
19645 static SDValue LowerSTRICT_FP_ROUND(SDValue Op, SelectionDAG &DAG) {
19645 static SDValue LowerSTRICT_FP_ROUND(SDValue Op, SelectionDAG &DAG) {
19654 static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
19654 static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
19657 SDValue LHS = Op.getOperand(0);
19658 SDValue RHS = Op.getOperand(1);
19698 SDValue X = LHS.getOperand(0);
19719 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
19726 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
19726 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
19740 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
19740 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
19780 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
19782 SDValue Op0 = Op.getOperand(0);
19787 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
19795 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
19800 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
19800 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
19801 SDValue Mag = Op.getOperand(0);
19802 SDValue Sign = Op.getOperand(1);
19836 SDValue SignMask = DAG.getConstantFP(
19838 SDValue MagMask = DAG.getConstantFP(
19844 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
19849 SDValue MagBits;
19862 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
19867 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
19867 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
19868 SDValue N0 = Op.getOperand(0);
19878 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
19886 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
19886 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
19894 static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
19895 SmallVectorImpl<SDValue> &SrcOps) {
19896 SmallVector<SDValue, 8> Opnds;
19897 DenseMap<SDValue, APInt> SrcOpMap;
19908 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
19923 SDValue Idx = I->getOperand(1);
19927 SDValue Src = I->getOperand(0);
19928 DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
19948 for (DenseMap<SDValue, APInt>::const_iterator I = SrcOpMap.begin(),
19959 static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
19959 static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
19961 SelectionDAG &DAG, SDValue &X86CC) {
19967 SmallVector<SDValue, 8> VecIns;
19987 SDValue LHS = VecIns[Slot];
19988 SDValue RHS = VecIns[Slot + 1];
19998 static bool hasNonFlagsUse(SDValue Op) {
20018 static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
20018 static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
20062 SDValue ArithOp = Op;
20125 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
20127 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
20134 SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
20134 SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
20134 SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
20162 SDValue In = Op0.getOperand(0);
20168 SDValue In = Op1.getOperand(0);
20183 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
20188 SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
20188 SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
20203 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp);
20204 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
20205 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW,
20207 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl);
20215 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
20229 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
20229 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
20262 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
20262 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
20304 SDValue
20334 SDValue N0 = N->getOperand(0);
20335 SDValue Zero = DAG.getConstant(0, DL, VT);
20337 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
20340 SDValue Cmp = DAG.getSetCC(DL, MVT::i8, N0, Zero, ISD::SETLT);
20341 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
20342 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
20349 SDValue SRA =
20363 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
20363 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
20365 SDValue &X86CC) {
20367 SDValue Op0 = And.getOperand(0);
20368 SDValue Op1 = And.getOperand(1);
20374 SDValue Src, BitNo;
20394 SDValue AndLHS = Op0;
20444 static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
20445 SDValue &Op1) {
20489 static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
20489 static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
20497 SDValue CC = Op.getOperand(2);
20500 SDValue LHS = Op.getOperand(0);
20501 SDValue LHS1 = extract128BitVector(LHS, 0, DAG, dl);
20502 SDValue LHS2 = extract128BitVector(LHS, NumElems / 2, DAG, dl);
20505 SDValue RHS = Op.getOperand(1);
20506 SDValue RHS1 = extract128BitVector(RHS, 0, DAG, dl);
20507 SDValue RHS2 = extract128BitVector(RHS, NumElems / 2, DAG, dl);
20517 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
20517 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
20519 SDValue Op0 = Op.getOperand(0);
20520 SDValue Op1 = Op.getOperand(1);
20521 SDValue CC = Op.getOperand(2);
20543 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc) {
20543 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc) {
20551 SmallVector<SDValue, 8> NewVecC;
20573 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
20573 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
20573 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
20595 SDValue ULEOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/false);
20606 SDValue UGEOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/true);
20621 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
20626 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
20626 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
20628 SDValue Op0 = Op.getOperand(0);
20629 SDValue Op1 = Op.getOperand(1);
20630 SDValue CC = Op.getOperand(2);
20656 SDValue Cmp;
20673 SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
20675 SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
20743 SDValue BC0 = peekThroughBitcasts(Op0);
20766 SDValue Result = Op0.getOperand(0);
20808 if (SDValue UGTOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/true)) {
20815 if (SDValue ULTOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/false)) {
20830 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
20841 if (SDValue V = LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
20866 SDValue SB;
20880 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
20881 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
20886 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
20887 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
20888 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
20890 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
20909 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
20913 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
20927 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
20933 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
20943 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
20943 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
20943 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
20946 SDValue &X86CC) {
20980 SDValue LHS = Op0.getOperand(0);
20981 SDValue RHS = Op0.getOperand(1);
20987 SDValue LHS = Op0;
20988 SDValue RHS = Op0;
21000 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
21000 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
21000 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
21003 SDValue &X86CC) const {
21010 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CC))
21018 if (SDValue PTEST = LowerVectorAllZeroTest(Op0, CC, Subtarget, DAG, X86CC))
21023 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
21051 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG);
21057 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
21057 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
21064 SDValue Op0 = Op.getOperand(0);
21065 SDValue Op1 = Op.getOperand(1);
21082 SDValue X86CC;
21083 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
21090 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
21090 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
21091 SDValue LHS = Op.getOperand(0);
21092 SDValue RHS = Op.getOperand(1);
21093 SDValue Carry = Op.getOperand(2);
21094 SDValue Cond = Op.getOperand(3);
21107 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
21115 static std::pair<SDValue, SDValue>
21115 static std::pair<SDValue, SDValue>
21116 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
21118 SDValue Value, Overflow;
21119 SDValue LHS = Op.getOperand(0);
21120 SDValue RHS = Op.getOperand(1);
21161 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
21161 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
21168 SDValue Value, Overflow;
21171 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
21177 static bool isX86LogicalCmp(SDValue Op) {
21191 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
21195 SDValue VOp0 = V.getOperand(0);
21201 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
21201 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
21203 SDValue Cond = Op.getOperand(0);
21204 SDValue Op1 = Op.getOperand(1);
21205 SDValue Op2 = Op.getOperand(2);
21208 SDValue CC;
21217 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
21222 SDValue Cmp =
21230 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
21250 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
21251 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
21252 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
21257 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
21262 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
21263 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
21270 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
21277 SDValue Op1Lo = extractSubVector(Op1, 0, DAG, DL, 32);
21278 SDValue Op2Lo = extractSubVector(Op2, 0, DAG, DL, 32);
21279 SDValue Op1Hi = extractSubVector(Op1, 32, DAG, DL, 32);
21280 SDValue Op2Hi = extractSubVector(Op2, 32, DAG, DL, 32);
21281 SDValue Lo = DAG.getSelect(DL, MVT::v32i1, Cond, Op1Lo, Op2Lo);
21282 SDValue Hi = DAG.getSelect(DL, MVT::v32i1, Cond, Op1Hi, Op2Hi);
21287 SDValue Op1Scalar;
21292 SDValue Op2Scalar;
21298 SDValue newSelect = DAG.getSelect(DL, Op1Scalar.getValueType(), Cond,
21302 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, newSelect);
21309 if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
21328 SDValue Cmp = Cond.getOperand(1);
21333 SDValue Y = isAllOnesConstant(Op2) ? Op1 : Op2;
21334 SDValue CmpOp0 = Cmp.getOperand(0);
21341 SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
21342 SDValue CmpZero = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Zero, CmpOp0);
21353 SDValue Zero = DAG.getConstant(0, DL, Op.getValueType());
21354 SDValue Res = // Res = 0 or -1.
21366 SDValue CmpOp0 = Cmp.getOperand(0);
21367 SDValue Src1, Src2;
21383 SDValue Neg;
21395 SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
21397 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
21416 SDValue Cmp = Cond.getOperand(1);
21430 SDValue Value;
21446 SDValue BTCC;
21447 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, BTCC)) {
21472 SDValue Res =
21486 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
21490 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
21507 SDValue Ops[] = { Op2, Op1, CC, Cond };
21508 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
21514 SDValue Ops[] = { Op2, Op1, CC, Cond };
21518 static SDValue LowerSIGN_EXTEND_Mask(SDValue Op,
21518 static SDValue LowerSIGN_EXTEND_Mask(SDValue Op,
21522 SDValue In = Op->getOperand(0);
21550 SDValue V;
21556 SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
21557 SDValue Zero = DAG.getConstant(0, dl, WideVT);
21575 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
21575 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
21577 SDValue In = Op->getOperand(0);
21591 static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
21591 static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
21594 SDValue In = Op->getOperand(0);
21653 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
21654 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
21664 SDValue Curr = In;
21665 SDValue SignExt = Curr;
21694 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
21695 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
21703 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
21703 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
21706 SDValue In = Op->getOperand(0);
21747 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
21754 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
21761 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
21762 SDValue StoredVal = Store->getValue();
21779 SDValue Value0 = extractSubVector(StoredVal, 0, DAG, DL, HalfSize);
21780 SDValue Value1 = extractSubVector(StoredVal, NumElems / 2, DAG, DL, HalfSize);
21781 SDValue Ptr0 = Store->getBasePtr();
21782 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, HalfAlign, DL);
21784 SDValue Ch0 =
21787 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
21796 static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
21798 SDValue StoredVal = Store->getValue();
21815 SmallVector<SDValue, 4> Stores;
21818 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), Offset, DL);
21819 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
21821 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
21830 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
21830 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
21834 SDValue StoredVal = St->getValue();
21865 SmallVector<SDValue, 4> CatOps;
21896 SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
21908 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
21908 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
21925 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
21932 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
21944 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
21956 static bool isXor1OfSetCC(SDValue Op) {
21965 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
21965 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
21967 SDValue Chain = Op.getOperand(0);
21968 SDValue Cond = Op.getOperand(1);
21969 SDValue Dest = Op.getOperand(2);
21971 SDValue CC;
21988 if (SDValue NewCond = LowerSETCC(Cond, DAG))
22014 SDValue Cmp = Cond.getOperand(1);
22037 SDValue Value;
22049 SDValue Cmp = Cond.getOperand(0).getOperand(1);
22081 SDValue FalseBB = User->getOperand(1);
22122 SDValue FalseBB = User->getOperand(1);
22129 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
22145 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
22165 SDValue BTCC;
22166 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, dl, DAG, BTCC)) {
22190 SDValue
22191 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
22202 SDValue Chain = Op.getOperand(0);
22203 SDValue Size = Op.getOperand(1);
22214 SDValue Result;
22221 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
22256 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
22271 SDValue Ops[2] = {Result, Chain};
22275 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
22275 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
22287 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
22297 SmallVector<SDValue, 8> MemOps;
22298 SDValue FIN = Op.getOperand(1);
22300 SDValue Store = DAG.getStore(
22316 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
22324 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
22332 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
22332 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
22342 SDValue Chain = Op.getOperand(0);
22343 SDValue SrcPtr = Op.getOperand(1);
22375 SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32),
22379 SDValue VAARG = DAG.getMemIntrinsicNode(
22391 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
22391 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
22401 SDValue Chain = Op.getOperand(0);
22402 SDValue DstPtr = Op.getOperand(1);
22403 SDValue SrcPtr = Op.getOperand(2);
22435 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
22436 SDValue SrcOp, uint64_t ShiftAmt,
22463 SmallVector<SDValue, 8> Elts;
22470 SDValue CurrentOp = SrcOp->getOperand(i);
22482 SDValue CurrentOp = SrcOp->getOperand(i);
22494 SDValue CurrentOp = SrcOp->getOperand(i);
22515 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
22516 SDValue SrcOp, SDValue ShAmt,
22516 SDValue SrcOp, SDValue ShAmt,
22555 SDValue ByteShift = DAG.getTargetConstant(
22569 SDValue ShOps[4] = {ShAmt, DAG.getConstant(0, dl, SVT), DAG.getUNDEF(SVT),
22585 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
22585 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
22600 SDValue Lo, Hi;
22624 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
22624 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
22624 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
22625 SDValue PreservedSrc,
22636 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
22650 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
22650 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
22650 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
22651 SDValue PreservedSrc,
22663 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
22699 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
22700 SDValue EntryEBP) {
22718 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
22719 SDValue ParentFrameOffset =
22732 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
22737 SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
22737 SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
22788 SDValue Rnd = Op.getOperand(2);
22800 SDValue Sae = Op.getOperand(2);
22813 SDValue Src2 = Op.getOperand(2);
22820 SDValue Rnd = Op.getOperand(3);
22834 SDValue Sae = Op.getOperand(3);
22849 SDValue Src1 = Op.getOperand(1);
22850 SDValue Src2 = Op.getOperand(2);
22851 SDValue Src3 = Op.getOperand(3);
22858 SDValue Rnd = Op.getOperand(4);
22875 SDValue Src = Op.getOperand(1);
22876 SDValue PassThru = Op.getOperand(2);
22877 SDValue Mask = Op.getOperand(3);
22883 SDValue Rnd = Op.getOperand(4);
22897 SDValue Src = Op.getOperand(1);
22898 SDValue PassThru = Op.getOperand(2);
22899 SDValue Mask = Op.getOperand(3);
22900 SDValue Rnd = Op.getOperand(4);
22914 SDValue Src1 = Op.getOperand(1);
22915 SDValue Src2 = Op.getOperand(2);
22916 SDValue passThru = Op.getOperand(3);
22917 SDValue Mask = Op.getOperand(4);
22925 SDValue Rnd = Op.getOperand(5);
22942 SDValue RoundingMode = Op.getOperand(5);
22945 SDValue Sae = Op.getOperand(6);
22956 SDValue Src1 = Op.getOperand(1);
22957 SDValue Src2 = Op.getOperand(2);
22958 SDValue passThru = Op.getOperand(3);
22959 SDValue Mask = Op.getOperand(4);
22960 SDValue Rnd = Op.getOperand(5);
22962 SDValue NewOp;
22975 SDValue Src1 = Op.getOperand(1);
22976 SDValue Src2 = Op.getOperand(2);
22977 SDValue passThru = Op.getOperand(3);
22978 SDValue Mask = Op.getOperand(4);
22979 SDValue Sae = Op.getOperand(5);
22992 SDValue Src1 = Op.getOperand(1);
22993 SDValue Src2 = Op.getOperand(2);
22994 SDValue PassThru = Op.getOperand(3);
22995 SDValue Mask = Op.getOperand(4);
22996 SDValue NewOp;
22998 SDValue Rnd = Op.getOperand(5);
23011 SDValue Src1 = Op.getOperand(1);
23012 SDValue Src2 = Op.getOperand(2);
23013 SDValue PassThru = Op.getOperand(3);
23014 SDValue Mask = Op.getOperand(4);
23018 SDValue Sae = Op.getOperand(5);
23029 SDValue Src1 = Op.getOperand(1);
23030 SDValue Src2 = Op.getOperand(2);
23031 SDValue Src3 = Op.getOperand(3);
23032 SDValue PassThru = Op.getOperand(4);
23033 SDValue Mask = Op.getOperand(5);
23034 SDValue Sae = Op.getOperand(6);
23047 SDValue Src1 = Op.getOperand(1);
23048 SDValue Src2 = Op.getOperand(2);
23049 SDValue Src3 = Op.getOperand(3);
23050 SDValue PassThru = Op.getOperand(4);
23051 SDValue Mask = Op.getOperand(5);
23055 SDValue Sae = Op.getOperand(6);
23065 SDValue Src1 = Op.getOperand(1);
23066 SDValue Src2 = Op.getOperand(2);
23067 SDValue Src3 = Op.getOperand(3);
23076 SDValue Src1 = Op.getOperand(1);
23077 SDValue Src2 = Op.getOperand(2);
23088 SDValue Src1 = Op.getOperand(1);
23089 SDValue Imm = Op.getOperand(2);
23090 SDValue Mask = Op.getOperand(3);
23091 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
23092 SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
23096 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
23104 SDValue CC = Op.getOperand(3);
23109 SDValue Sae = Op.getOperand(4);
23121 SDValue Src1 = Op.getOperand(1);
23122 SDValue Src2 = Op.getOperand(2);
23123 SDValue CC = Op.getOperand(3);
23124 SDValue Mask = Op.getOperand(4);
23126 SDValue Cmp;
23128 SDValue Sae = Op.getOperand(5);
23138 SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
23142 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
23149 SDValue LHS = Op.getOperand(1);
23150 SDValue RHS = Op.getOperand(2);
23151 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
23152 SDValue InvComi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, RHS, LHS);
23153 SDValue SetCC;
23157 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
23163 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
23186 SDValue LHS = Op.getOperand(1);
23187 SDValue RHS = Op.getOperand(2);
23189 SDValue Sae = Op.getOperand(4);
23191 SDValue FCmp;
23202 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
23213 SDValue Mask = Op.getOperand(3);
23214 SDValue DataToCompress = Op.getOperand(1);
23215 SDValue PassThru = Op.getOperand(2);
23228 SDValue Src1 = Op.getOperand(1);
23229 SDValue Src2 = Op.getOperand(2);
23230 SDValue Src3 = Op.getOperand(3);
23231 SDValue Imm = Op.getOperand(4);
23232 SDValue Mask = Op.getOperand(5);
23233 SDValue Passthru = (IntrData->Type == FIXUPIMM)
23239 SDValue Sae = Op.getOperand(6);
23246 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
23258 SDValue RoundingMode =
23268 SDValue RoundingMode =
23279 SDValue Control = DAG.getConstant(Imm, dl, Op.getValueType());
23288 SDValue Res;
23295 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
23300 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
23301 SDValue Results[] = { SetCC, Res };
23308 SDValue Src = Op.getOperand(1);
23309 SDValue PassThru = Op.getOperand(2);
23310 SDValue Mask = Op.getOperand(3);
23322 SDValue Src = Op.getOperand(1);
23323 SDValue Rnd = Op.getOperand(2);
23324 SDValue PassThru = Op.getOperand(3);
23325 SDValue Mask = Op.getOperand(4);
23338 SDValue Src = Op.getOperand(1);
23339 SDValue PassThru = Op.getOperand(2);
23340 SDValue Mask = Op.getOperand(3);
23443 SDValue LHS = Op.getOperand(1);
23444 SDValue RHS = Op.getOperand(2);
23445 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
23446 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
23505 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
23507 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
23508 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
23520 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
23533 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
23552 SDValue Op1 = Op.getOperand(1);
23559 SDValue Result = DAG.getMCSymbol(LSDASym, VT);
23564 SDValue FnOp = Op.getOperand(1);
23565 SDValue IncomingFPOp = Op.getOperand(2);
23603 SDValue Operation =
23607 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
23609 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
23622 SDValue ShAmt = Op.getOperand(2);
23671 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
23671 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
23672 SDValue Src, SDValue Mask, SDValue Base,
23672 SDValue Src, SDValue Mask, SDValue Base,
23672 SDValue Src, SDValue Mask, SDValue Base,
23673 SDValue Index, SDValue ScaleOp, SDValue Chain,
23673 SDValue Index, SDValue ScaleOp, SDValue Chain,
23673 SDValue Index, SDValue ScaleOp, SDValue Chain,
23681 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
23693 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
23694 SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
23699 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
23699 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
23700 SDValue Src, SDValue Mask, SDValue Base,
23700 SDValue Src, SDValue Mask, SDValue Base,
23700 SDValue Src, SDValue Mask, SDValue Base,
23701 SDValue Index, SDValue ScaleOp, SDValue Chain,
23701 SDValue Index, SDValue ScaleOp, SDValue Chain,
23701 SDValue Index, SDValue ScaleOp, SDValue Chain,
23710 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
23730 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
23731 SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
23736 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
23736 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
23737 SDValue Src, SDValue Mask, SDValue Base,
23737 SDValue Src, SDValue Mask, SDValue Base,
23737 SDValue Src, SDValue Mask, SDValue Base,
23738 SDValue Index, SDValue ScaleOp, SDValue Chain,
23738 SDValue Index, SDValue ScaleOp, SDValue Chain,
23738 SDValue Index, SDValue ScaleOp, SDValue Chain,
23746 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
23760 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
23761 SDValue Res = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
23766 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
23766 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
23767 SDValue Mask, SDValue Base, SDValue Index,
23767 SDValue Mask, SDValue Base, SDValue Index,
23767 SDValue Mask, SDValue Base, SDValue Index,
23768 SDValue ScaleOp, SDValue Chain,
23768 SDValue ScaleOp, SDValue Chain,
23776 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
23778 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
23779 SDValue Segment = DAG.getRegister(0, MVT::i32);
23782 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
23783 SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
23796 static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
23801 SmallVectorImpl<SDValue> &Results) {
23802 SDValue Chain = N->getOperand(0);
23803 SDValue Glue;
23812 SDValue N1Ops[] = {Chain, Glue};
23818 SDValue LO, HI;
23833 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
23841 SDValue Ops[] = { LO, HI };
23842 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
23854 SmallVectorImpl<SDValue> &Results) {
23858 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
23864 SDValue Chain = Results[1];
23867 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
23872 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
23872 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
23874 SmallVector<SDValue, 3> Results;
23881 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
23881 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
23883 SDValue Chain = Op.getOperand(0);
23884 SDValue RegNode = Op.getOperand(2);
23899 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
23899 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
23901 SDValue Chain = Op.getOperand(0);
23902 SDValue EHGuard = Op.getOperand(2);
23918 static SDValue
23919 EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val,
23919 EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val,
23920 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
23924 SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
23925 SDValue Ops[] = { Chain, Val, Ptr, Undef };
23932 static SDValue
23933 EmitMaskedTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl,
23934 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
23934 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
23934 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
23938 SDValue Ops[] = { Chain, Val, Ptr, Mask };
23944 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
23944 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
23987 SDValue Chain = Op->getOperand(0);
24005 SDValue Operation =
24008 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
24015 SDValue Chain = Op.getOperand(0);
24027 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
24029 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
24044 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
24048 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
24052 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
24059 SDValue Chain = Op.getOperand(0);
24060 SDValue Src = Op.getOperand(2);
24061 SDValue Base = Op.getOperand(3);
24062 SDValue Index = Op.getOperand(4);
24063 SDValue Mask = Op.getOperand(5);
24064 SDValue Scale = Op.getOperand(6);
24070 SDValue Chain = Op.getOperand(0);
24071 SDValue Src = Op.getOperand(2);
24072 SDValue Base = Op.getOperand(3);
24073 SDValue Index = Op.getOperand(4);
24074 SDValue Mask = Op.getOperand(5);
24075 SDValue Scale = Op.getOperand(6);
24081 SDValue Chain = Op.getOperand(0);
24082 SDValue Base = Op.getOperand(2);
24083 SDValue Mask = Op.getOperand(3);
24084 SDValue Index = Op.getOperand(4);
24085 SDValue Src = Op.getOperand(5);
24086 SDValue Scale = Op.getOperand(6);
24095 SDValue Chain = Op.getOperand(0);
24096 SDValue Mask = Op.getOperand(2);
24097 SDValue Index = Op.getOperand(3);
24098 SDValue Base = Op.getOperand(4);
24099 SDValue Scale = Op.getOperand(5);
24105 SmallVector<SDValue, 2> Results;
24114 SmallVector<SDValue, 2> Results;
24126 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
24128 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
24129 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
24136 SDValue Mask = Op.getOperand(4);
24137 SDValue DataToTruncate = Op.getOperand(3);
24138 SDValue Addr = Op.getOperand(2);
24139 SDValue Chain = Op.getOperand(0);
24154 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24167 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24179 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
24179 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
24192 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
24194 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
24201 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
24206 SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
24206 SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
24212 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
24212 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
24243 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
24283 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
24283 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
24308 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
24308 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
24309 SDValue Chain = Op.getOperand(0);
24310 SDValue Offset = Op.getOperand(1);
24311 SDValue Handler = Op.getOperand(2);
24320 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
24323 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
24334 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
24334 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
24352 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
24352 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
24359 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
24359 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
24366 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
24366 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
24370 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
24370 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
24372 SDValue Root = Op.getOperand(0);
24373 SDValue Trmp = Op.getOperand(1); // trampoline
24374 SDValue FPtr = Op.getOperand(2); // nested function
24375 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
24382 SDValue OutChains[6];
24395 SDValue Addr = Trmp;
24481 SDValue OutChains[4];
24482 SDValue Addr, Disp;
24518 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
24518 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
24547 SDValue StackSlot =
24554 SDValue Ops[] = { DAG.getEntryNode(), StackSlot };
24555 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
24560 SDValue CWD =
24564 SDValue CWD1 =
24569 SDValue CWD2 =
24575 SDValue RetVal =
24587 static SDValue LowerVectorIntUnary(SDValue Op, SelectionDAG &DAG) {
24587 static SDValue LowerVectorIntUnary(SDValue Op, SelectionDAG &DAG) {
24592 SDValue Src = Op.getOperand(0);
24598 SDValue Lo = extractSubVector(Src, 0, DAG, dl, SizeInBits / 2);
24599 SDValue Hi = extractSubVector(Src, NumElems / 2, DAG, dl, SizeInBits / 2);
24608 static SDValue Lower256IntUnary(SDValue Op, SelectionDAG &DAG) {
24608 static SDValue Lower256IntUnary(SDValue Op, SelectionDAG &DAG) {
24616 static SDValue Lower512IntUnary(SDValue Op, SelectionDAG &DAG) {
24616 static SDValue Lower512IntUnary(SDValue Op, SelectionDAG &DAG) {
24629 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
24629 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
24651 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
24652 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
24653 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
24659 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
24659 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
24673 SmallVector<SDValue, 64> LUTVec;
24676 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
24683 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
24684 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
24686 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
24687 SDValue Lo = Op0;
24688 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
24689 SDValue HiZ;
24701 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
24713 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
24730 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
24731 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
24732 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
24741 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
24741 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
24763 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
24763 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
24787 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
24802 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
24802 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
24806 SDValue N0 = Op.getOperand(0);
24817 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
24825 static SDValue split256IntArith(SDValue Op, SelectionDAG &DAG) {
24825 static SDValue split256IntArith(SDValue Op, SelectionDAG &DAG) {
24835 SDValue LHS = Op.getOperand(0);
24836 SDValue LHS1 = extract128BitVector(LHS, 0, DAG, dl);
24837 SDValue LHS2 = extract128BitVector(LHS, NumElems / 2, DAG, dl);
24840 SDValue RHS = Op.getOperand(1);
24841 SDValue RHS1 = extract128BitVector(RHS, 0, DAG, dl);
24842 SDValue RHS2 = extract128BitVector(RHS, NumElems / 2, DAG, dl);
24854 static SDValue split512IntArith(SDValue Op, SelectionDAG &DAG) {
24854 static SDValue split512IntArith(SDValue Op, SelectionDAG &DAG) {
24864 SDValue LHS = Op.getOperand(0);
24865 SDValue LHS1 = extract256BitVector(LHS, 0, DAG, dl);
24866 SDValue LHS2 = extract256BitVector(LHS, NumElems / 2, DAG, dl);
24869 SDValue RHS = Op.getOperand(1);
24870 SDValue RHS1 = extract256BitVector(RHS, 0, DAG, dl);
24871 SDValue RHS2 = extract256BitVector(RHS, NumElems / 2, DAG, dl);
24881 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
24881 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
24897 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
24897 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
24900 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
24925 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, X, Y);
24926 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Add, ISD::SETUGT);
24931 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
24932 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
24945 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
24945 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
24952 SDValue N0 = Op.getOperand(0);
24953 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
24955 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_GE, DL, MVT::i8),
24963 SDValue Src = Op.getOperand(0);
24964 SDValue Sub =
24979 static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) {
24979 static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) {
24988 SDValue N0 = Op.getOperand(0);
24989 SDValue N1 = Op.getOperand(1);
24996 SDValue Sign = DAG.getConstant(APInt::getSignedMinValue(16), DL, VT);
25000 SDValue Result = DAG.getNode(Opcode, DL, VT, N0, N1);
25014 SDValue Cond = DAG.getSetCC(DL, VT, N0, N1, CC);
25018 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
25018 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
25030 SDValue A = Op.getOperand(0);
25031 SDValue B = Op.getOperand(1);
25054 SDValue Undef = DAG.getUNDEF(VT);
25055 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
25056 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
25058 SDValue BLo, BHi;
25061 SmallVector<SDValue, 16> LoOps, HiOps;
25079 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
25080 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
25093 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
25094 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
25097 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
25101 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
25138 SDValue Zero = DAG.getConstant(0, dl, VT);
25141 SDValue AloBlo = Zero;
25145 SDValue AloBhi = Zero;
25147 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
25151 SDValue AhiBlo = Zero;
25153 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
25157 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
25163 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
25163 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
25169 SDValue A = Op.getOperand(0);
25170 SDValue B = Op.getOperand(1);
25196 SDValue Odd0 = DAG.getVectorShuffle(VT, dl, A, A,
25199 SDValue Odd1 = DAG.getVectorShuffle(VT, dl, B, B,
25209 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
25214 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
25223 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
25228 SDValue Zero = DAG.getConstant(0, dl, VT);
25229 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
25231 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
25234 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
25256 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
25257 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
25258 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
25271 SDValue ALo = extract128BitVector(A, 0, DAG, dl);
25272 SDValue BLo = extract128BitVector(B, 0, DAG, dl);
25273 SDValue AHi = extract128BitVector(A, NumElts / 2, DAG, dl);
25274 SDValue BHi = extract128BitVector(B, NumElts / 2, DAG, dl);
25279 SDValue Lo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
25280 SDValue Hi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
25311 SDValue ALo, AHi;
25330 SDValue BLo, BHi;
25333 SmallVector<SDValue, 16> LoOps, HiOps;
25336 SDValue LoOp = B.getOperand(i + j);
25337 SDValue HiOp = B.getOperand(i + j + 8);
25374 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
25375 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
25383 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
25383 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
25402 SDValue InChain = DAG.getEntryNode();
25410 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
25421 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
25435 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
25435 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
25486 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
25486 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
25490 SDValue R = Op.getOperand(0);
25491 SDValue Amt = Op.getOperand(1);
25497 SDValue Ex = DAG.getBitcast(ExVT, R);
25508 SDValue Upper =
25510 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
25519 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
25521 SDValue Lower =
25564 SDValue Zeros = DAG.getConstant(0, dl, VT);
25567 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
25579 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
25588 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
25597 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
25599 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
25610 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
25610 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
25614 SDValue R = Op.getOperand(0);
25615 SDValue Amt = Op.getOperand(1);
25620 if (SDValue BaseShAmt = DAG.getSplatValue(Amt)) {
25646 SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
25656 SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
25665 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
25682 std::vector<SDValue> Vals(Ratio);
25698 static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
25698 static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
25708 SmallVector<SDValue, 8> Elts;
25715 SDValue Op = Amt->getOperand(i);
25745 SDValue Z = DAG.getConstant(0, dl, VT);
25746 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
25747 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
25761 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
25761 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
25765 SDValue R = Op.getOperand(0);
25766 SDValue Amt = Op.getOperand(1);
25777 if (SDValue V = LowerScalarImmediateShift(Op, DAG, Subtarget))
25780 if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget))
25791 SDValue Zero = DAG.getConstant(0, dl, VT);
25804 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
25805 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
25806 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
25807 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
25816 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
25817 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
25836 SDValue Amt1, Amt2;
25840 SDValue A = Amt->getOperand(i);
25868 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
25870 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
25880 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
25887 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
25888 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
25889 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
25890 SDValue Zero = DAG.getConstant(0, dl, VT);
25891 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
25892 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
25906 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
25907 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
25908 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
25909 SDValue Amt0 =
25911 SDValue Amt1 =
25913 SDValue Sra1 =
25915 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
25927 SDValue Amt0, Amt1, Amt2, Amt3;
25939 SDValue Z = DAG.getConstant(0, dl, VT);
25945 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
25946 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
25960 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
25961 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
25962 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
25963 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
25968 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
25969 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
25972 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
25973 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
26004 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
26023 SmallVector<SDValue, 16> LoAmt, HiAmt;
26032 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
26033 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
26035 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
26036 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
26074 SDValue Z = DAG.getConstant(0, dl, SelVT);
26075 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
26088 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
26111 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26112 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26113 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
26114 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
26121 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
26122 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
26156 SDValue Z = DAG.getConstant(0, dl, VT);
26157 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
26158 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
26159 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
26160 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
26165 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
26166 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
26191 SDValue C =
26209 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
26242 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
26242 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
26248 SDValue R = Op.getOperand(0);
26249 SDValue Amt = Op.getOperand(1);
26340 SDValue Z = DAG.getConstant(0, DL, SelVT);
26341 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
26353 SDValue M;
26392 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
26394 SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);
26395 SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);
26400 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
26405 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
26406 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
26415 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
26416 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
26418 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
26421 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
26605 static SDValue emitLockedStackOp(SelectionDAG &DAG,
26607 SDValue Chain, SDLoc DL) {
26637 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
26638 SDValue Ops[] = {
26651 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
26652 SDValue Ops[] = {
26666 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
26666 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
26681 SDValue Chain = Op.getOperand(0);
26689 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
26689 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
26705 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
26707 SDValue Ops[] = { cpIn.getValue(0),
26714 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
26717 SDValue cpOut =
26719 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
26721 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
26728 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
26728 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
26733 SDValue Lo, Hi;
26744 SDValue Lo, Hi;
26756 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
26756 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
26758 SDValue Src = Op.getOperand(0);
26768 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,
26771 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,
26781 SDValue Lo, Hi;
26794 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
26838 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
26838 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
26854 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
26866 SDValue Zeros = DAG.getConstant(0, DL, VT);
26867 SDValue V32 = DAG.getBitcast(VT, V);
26868 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
26869 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
26895 SDValue ShifterV = DAG.getConstant(8, DL, VT);
26896 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
26902 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
26902 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
26926 SmallVector<SDValue, 64> LUTVec;
26929 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
26930 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
26933 SDValue FourV = DAG.getConstant(4, DL, VT);
26934 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
26937 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
26942 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
26943 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
26949 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
26949 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
26955 SDValue Op0 = Op.getOperand(0);
26981 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
26982 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
26993 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
26993 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
27000 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
27000 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
27002 SDValue In = Op.getOperand(0);
27009 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
27029 SmallVector<SDValue, 16> MaskElts;
27038 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
27039 SDValue Res = DAG.getBitcast(MVT::v16i8, In);
27045 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
27045 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
27054 SDValue In = Op.getOperand(0);
27075 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
27076 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
27077 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
27090 SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
27096 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
27097 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
27103 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
27103 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
27135 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
27135 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
27138 SDValue Chain = N->getOperand(0);
27139 SDValue LHS = N->getOperand(1);
27140 SDValue RHS = N->getOperand(2);
27177 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
27184 SDValue NewChain = DAG.getNode(X86ISD::MEMBARRIER, DL, MVT::Other, Chain);
27191 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
27199 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
27199 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
27222 SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
27225 SDValue Ops[] = { Node->getChain(), SclToVec, Node->getBasePtr() };
27226 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys,
27242 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
27250 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
27250 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
27262 SDValue Carry = Op.getOperand(2);
27269 SDValue Sum = DAG.getNode(Opc, DL, VTs, Op.getOperand(0),
27272 SDValue SetCC = getSETCC(X86::COND_B, Sum.getValue(1), DL, DAG);
27279 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
27279 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
27287 SDValue Arg = Op.getOperand(0);
27307 SDValue Callee =
27318 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
27318 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
27325 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
27327 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
27335 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
27335 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
27356 SDValue N1 = InOp.getOperand(1);
27366 SmallVector<SDValue, 16> Ops;
27372 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
27378 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) :
27384 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
27384 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
27390 SDValue Src = N->getValue();
27395 SDValue Scale = N->getScale();
27396 SDValue Index = N->getIndex();
27397 SDValue Mask = N->getMask();
27398 SDValue Chain = N->getChain();
27399 SDValue BasePtr = N->getBasePtr();
27409 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
27410 SDValue NewScatter = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
27444 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
27445 SDValue NewScatter = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
27450 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
27450 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
27456 SDValue Mask = N->getMask();
27458 SDValue PassThru = N->getPassThru();
27467 SDValue NewLoad = DAG.getMaskedLoad(VT, dl, N->getChain(),
27474 SDValue Select = DAG.getNode(ISD::VSELECT, dl, MaskVT, Mask, NewLoad,
27506 SDValue NewLoad = DAG.getMaskedLoad(WideDataVT, dl, N->getChain(),
27512 SDValue Exract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,
27515 SDValue RetOps[] = {Exract, NewLoad.getValue(1)};
27519 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
27519 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
27522 SDValue DataToStore = N->getValue();
27525 SDValue Mask = N->getMask();
27560 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
27560 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
27568 SDValue Index = N->getIndex();
27569 SDValue Mask = N->getMask();
27570 SDValue PassThru = N->getPassThru();
27600 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
27602 SDValue NewGather = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
27605 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT,
27610 SDValue X86TargetLowering::LowerGC_TRANSITION_START(SDValue Op,
27610 SDValue X86TargetLowering::LowerGC_TRANSITION_START(SDValue Op,
27618 SmallVector<SDValue, 2> Ops;
27626 SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
27631 SDValue X86TargetLowering::LowerGC_TRANSITION_END(SDValue Op,
27631 SDValue X86TargetLowering::LowerGC_TRANSITION_END(SDValue Op,
27639 SmallVector<SDValue, 2> Ops;
27647 SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
27652 SDValue X86TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
27652 SDValue X86TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
27654 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
27660 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
27660 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
27791 SmallVectorImpl<SDValue> &Results,
27793 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
27818 SmallVectorImpl<SDValue>&Results,
27835 SDValue Wide =
27855 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
27856 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
27857 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
27860 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
27887 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
27889 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
27891 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
27893 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
27902 SDValue Lo, Hi, Tmp;
27929 SDValue UNDEF = DAG.getUNDEF(VT);
27930 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
27932 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
27951 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
27954 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
27955 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
27956 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
27966 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
27979 SDValue In = N->getOperand(0);
27990 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
27995 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, In,
28024 SDValue Lo, Hi;
28029 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
28047 SDValue In = N->getOperand(0);
28061 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
28062 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
28066 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
28069 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
28073 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28099 SDValue Lo = getExtendInVec(N->getOpcode(), dl, LoVT, In, DAG);
28108 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
28111 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28120 SDValue Src = N->getOperand(0);
28131 SDValue Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src);
28148 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
28171 SDValue Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
28193 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
28194 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
28203 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned))
28209 SDValue Src = N->getOperand(0);
28220 SDValue Src = N->getOperand(0);
28228 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
28229 SDValue VBias =
28231 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
28235 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
28242 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
28284 SDValue cpInL, cpInH;
28295 SDValue swapInL, swapInH;
28310 SDValue Result;
28326 SDValue RBXSave = DAG.getCopyFromReg(swapInH.getValue(0), dl,
28329 SDValue Ops[] = {/*Chain*/ RBXSave.getValue(1), N->getOperand(1), swapInL,
28339 SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
28343 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
28346 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
28349 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
28351 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
28353 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
28372 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
28373 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
28375 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
28386 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
28387 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD_FLAG,
28390 SDValue Chain = Result.getValue(1);
28391 SDValue InFlag = Result.getValue(2);
28397 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
28401 SDValue StoreOps[] = { Chain, Result, StackPtr, InFlag };
28445 SDValue Lo, Hi;
28449 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
28457 SDValue Lo, Hi;
28462 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, DstVT, Lo, Hi);
28471 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, WideVT, N->getOperand(0));
28483 SDValue Index = Gather->getIndex();
28489 SDValue Mask = Gather->getMask();
28491 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
28501 SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
28503 SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
28525 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
28528 SDValue Chain = Res.getValue(1);
28539 SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
28540 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
29065 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
29089 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
31423 X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
31476 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
31477 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
31481 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
31509 SDValue Src = Op.getOperand(0);
31620 SmallVector<SDValue, 2> Ops;
31668 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
31680 SDValue Src = Op.getOperand(0);
31708 SDValue Src = Op.getOperand(0);
31719 SDValue Src = Op.getOperand(0);
31758 SmallVector<SDValue, 2> Ops;
31805 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
31805 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
31816 SDValue &V1, const SDLoc &DL, SelectionDAG &DAG,
32067 SDValue &V1, SDValue &V2, const SDLoc &DL,
32067 SDValue &V1, SDValue &V2, const SDLoc &DL,
32133 bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
32133 bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
32248 SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
32249 SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
32274 static SDValue combineX86ShuffleChainWithExtract(
32275 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
32275 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
32288 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
32288 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
32288 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
32300 SDValue V1 = peekThroughBitcasts(Inputs[0]);
32301 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
32312 SDValue Res;
32342 SDValue Src = Inputs[0];
32460 SDValue NewV1 = V1; // Save operand in case early exit happens.
32485 SDValue NewV1 = V1; // Save operands in case early exit happens.
32486 SDValue NewV2 = V2;
32568 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
32592 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
32594 SDValue Zero = getZeroVector(MaskVT, Subtarget, DAG, DL);
32601 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
32618 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
32646 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
32660 SmallVector<SDValue, 16> VPermIdx;
32662 SDValue Idx =
32666 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
32701 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
32716 SmallVector<SDValue, 16> PSHUFBMask;
32735 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
32747 SmallVector<SDValue, 16> VPPERMMask;
32766 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
32773 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
32793 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
32812 static SDValue combineX86ShuffleChainWithExtract(
32813 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
32813 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
32821 SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end());
32828 SDValue &Src = WideInputs[i];
32865 for (SDValue &NewInput : WideInputs) {
32899 SDValue WideRoot = WideInputs[0];
32900 if (SDValue WideShuffle = combineX86ShuffleChain(
32913 static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops,
32913 static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops,
32914 ArrayRef<int> Mask, SDValue Root,
32930 SDValue SrcOp = Ops[i];
32991 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
33024 static SDValue combineX86ShufflesRecursively(
33025 ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
33025 ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
33039 SDValue Op = SrcOps[SrcOpIndex];
33054 SmallVector<SDValue, 2> OpInputs;
33065 SmallVector<SDValue, 16> Ops(SrcOps.begin(), SrcOps.end());
33069 SDValue InputBC = peekThroughBitcasts(Input);
33084 for (SDValue OpInput : OpInputs)
33199 if (SDValue Res = combineX86ShufflesRecursively(
33207 if (SDValue Cst = combineX86ShufflesConstants(
33243 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
33243 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
33257 SmallVector<SDValue, 2> Ops;
33298 static SDValue
33299 combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
33308 SmallVector<SDValue, 8> Chain;
33309 SDValue V = N.getOperand(0);
33398 SDValue W = Chain.pop_back_val();
33427 static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
33427 static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
33440 auto BC0 = peekThroughBitcasts(N.getOperand(0));
33441 auto BC1 = peekThroughBitcasts(N.getOperand(1));
33450 SDValue Lo, Hi;
33458 SDValue Horiz = DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
33465 SDValue Src = N.getOperand(0);
33466 SDValue BC = peekThroughBitcasts(Src);
33479 if (SDValue Res = combineX86ShufflesRecursively(
33518 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
33519 SDValue BcastLd =
33529 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
33539 SDValue N0 = N.getOperand(0);
33540 SDValue N1 = N.getOperand(1);
33564 SDValue N0 = N.getOperand(0);
33565 SDValue N1 = N.getOperand(1);
33569 SDValue Src = N0.getOperand(0);
33571 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
33584 SDValue N0 = N.getOperand(0);
33585 SDValue N1 = N.getOperand(1);
33593 SDValue N10 = N1.getOperand(0);
33594 SDValue N11 = N1.getOperand(1);
33600 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
33603 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
33604 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
33613 SDValue Op0 = N.getOperand(0);
33614 SDValue Op1 = N.getOperand(1);
33615 SDValue Op2 = N.getOperand(2);
33633 SmallVector<SDValue, 2> Ops1;
33654 SmallVector<SDValue, 2> Ops0;
33703 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
33706 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
33726 SDValue V = N.getOperand(0);
33757 SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
33786 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG))
33836 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
33836 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
33851 SDValue V1 = N->getOperand(0);
33852 SDValue V2 = N->getOperand(1);
33866 SDValue LHS, RHS;
33895 static SDValue combineShuffleToFMAddSub(SDNode *N,
33910 SDValue Op0 = N->getOperand(0);
33911 SDValue Op1 = N->getOperand(1);
33912 SDValue FMAdd = Op0, FMSub = Op1;
33938 static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N,
33941 if (SDValue V = combineShuffleToFMAddSub(N, Subtarget, DAG))
33944 SDValue Opnd0, Opnd1;
33953 SDValue Opnd2;
33974 static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
33991 SDValue N0 = N->getOperand(0);
33992 SDValue N1 = N->getOperand(1);
34011 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
34017 static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) {
34027 SDValue SrcOp = N->getOperand(0);
34035 SDValue HOp = N->getOperand(0);
34053 SDValue X;
34106 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
34136 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
34140 if (SDValue V = narrowShuffle(Shuf, DAG))
34149 if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
34152 if (SDValue HAddSub = foldShuffleOfHorizOp(N, DAG))
34157 if (SDValue LD = combineToConsecutiveLoads(VT, N, dl, DAG, Subtarget, true))
34166 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, DAG, Subtarget))
34170 SDValue Op(N, 0);
34171 if (SDValue Shuffle = combineTargetShuffle(Op, DAG, DCI, Subtarget))
34179 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
34198 SDValue In = N->getOperand(0).getOperand(0);
34225 SDValue In = N->getOperand(0).getOperand(1);
34226 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, dl, In.getValueType(), In);
34239 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
34240 SDValue VZLoad =
34255 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
34267 SDValue LHS = Op.getOperand(0);
34268 SDValue RHS = Op.getOperand(1);
34283 SDValue Amt = Op.getOperand(1);
34307 SDValue Src = Op.getOperand(0);
34316 SDValue Src = Op.getOperand(0);
34338 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
34355 SDValue Src = Op.getOperand(0);
34377 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
34395 SDValue Src = Op.getOperand(0);
34406 SDValue N0 = Op.getOperand(0);
34407 SDValue N1 = Op.getOperand(1);
34425 SDValue NewN0 = SimplifyMultipleUseDemandedBits(
34427 SDValue NewN1 = SimplifyMultipleUseDemandedBits(
34458 SDValue Src = Op.getOperand(0);
34491 SDValue Src = Op.getOperand(0);
34510 SDValue Mask = Op.getOperand(0);
34520 SDValue Mask = Op.getOperand(1);
34529 SDValue Mask = Op.getOperand(2);
34554 SDValue Ext0 =
34556 SDValue ExtOp =
34558 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
34559 SDValue Insert =
34566 SDValue Src = Op.getOperand(0);
34590 SDValue Ext0 =
34592 SDValue ExtOp =
34594 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
34595 SDValue Insert =
34607 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
34608 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
34609 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
34631 SDValue Ext0 =
34633 SDValue Ext1 =
34635 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ext0, Ext1);
34636 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
34637 SDValue Insert =
34647 SmallVector<SDValue, 2> OpInputs;
34708 SDValue Op, const APInt &OriginalDemandedBits,
34719 SDValue LHS = Op.getOperand(0);
34720 SDValue RHS = Op.getOperand(1);
34731 SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
34733 SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
34744 SDValue Op0 = Op.getOperand(0);
34745 SDValue Op1 = Op.getOperand(1);
34766 SDValue NewShift = TLO.DAG.getNode(
34809 SDValue Op0 = Op.getOperand(0);
34810 SDValue Op1 = Op.getOperand(1);
34825 SDValue Op00 = Op0.getOperand(0);
34860 SDValue Vec = Op.getOperand(0);
34886 if (SDValue V = SimplifyMultipleUseDemandedBits(
34898 SDValue Vec = Op.getOperand(0);
34899 SDValue Scl = Op.getOperand(1);
34955 SDValue Src = Op.getOperand(0);
34992 SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
34993 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
35003 SDValue Vec = Op.getOperand(0);
35015 SmallVector<SDValue, 2> ShuffleOps;
35063 static SDValue
35069 SDValue InVec = N->getOperand(0);
35070 SDValue EltNo = N->getOperand(1);
35098 SmallVector<SDValue, 2> ShuffleOps;
35129 SDValue LdNode = (Idx < (int)NumOriginalElts) ? ShuffleOps[0] : ShuffleOps[1];
35167 SDValue Shuffle = UnaryShuffle ? DAG.getUNDEF(OriginalVT)
35178 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size) {
35192 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
35193 SDValue Src, const SDLoc &DL) {
35214 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
35214 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
35295 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
35314 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
35314 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
35323 SDValue In = Op.getOperand(Idx);
35331 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
35344 SDValue Op = N->getOperand(0);
35363 SDValue LHS = Op.getOperand(0);
35364 SDValue RHS = Op.getOperand(1);
35387 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
35391 SDValue Splat = BV->getSplatValue();
35411 SmallVector<SDValue, 8> Ops;
35448 SDValue Intrin = DAG.getConstant(IntrinOp, DL, MVT::i32);
35458 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
35461 SDValue N0 = N->getOperand(0);
35473 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
35483 SDValue N00 = N0.getOperand(0);
35488 SDValue V = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32,
35516 SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
35520 SmallVector<SDValue, 4> Ops(N0->op_begin(), N0->op_end());
35529 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
35555 SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
35556 SDValue ResNode =
35589 SDValue Op = N0.getOperand(i);
35594 SDValue N00 = N0.getOperand(0);
35614 SDValue N00 = N0.getOperand(0);
35623 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
35650 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
35670 SDValue LogicOp0 = N0.getOperand(0);
35671 SDValue LogicOp1 = N0.getOperand(1);
35678 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
35685 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
35695 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
35695 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
35695 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
35696 SDValue AbsOp1 = Abs->getOperand(0);
35715 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
35715 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
35716 const SDValue &Zext1, const SDLoc &DL,
35725 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
35728 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
35730 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
35745 static SDValue combineHorizontalMinMaxResult(SDNode *Extract, SelectionDAG &DAG,
35757 SDValue Src = DAG.matchBinOpReduction(
35768 SDValue MinPos = Src;
35776 SDValue Lo = extractSubVector(MinPos, 0, DAG, DL, SubSizeInBits);
35777 SDValue Hi = extractSubVector(MinPos, NumSubElts, DAG, DL, SubSizeInBits);
35786 SDValue Mask;
35803 SDValue Upper = DAG.getVectorShuffle(
35822 static SDValue combineHorizontalPredicateResult(SDNode *Extract,
35837 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
35848 SDValue Movmsk;
35866 SDValue Lo, Hi;
35899 SDValue Lo, Hi;
35913 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
35923 SDValue Mask = DAG.getConstant(1, DL, CmpVT);
35924 SDValue Result = DAG.getNode(ISD::CTPOP, DL, CmpVT, Movmsk);
35929 SDValue CmpC;
35946 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
35947 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
35948 SDValue Zero = DAG.getConstant(0, DL, ExtractVT);
35952 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
35977 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
35998 SDValue Zext0, Zext1;
36004 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
36018 SDValue Shuffle =
36035 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
36042 SDValue Src = N->getOperand(0);
36043 SDValue Idx = N->getOperand(1);
36054 SDValue SrcBC = peekThroughBitcasts(Src);
36058 SDValue SrcOp = SrcBC.getOperand(0);
36070 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
36092 SmallVector<SDValue, 2> Ops;
36136 SDValue SrcOp = Ops[SrcIdx / Mask.size()];
36157 SDValue ExtOp = DAG.getNode(OpCode, dl, MVT::i32, SrcOp,
36167 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG) {
36169 SDValue Vec = ExtElt->getOperand(0);
36170 SDValue Index = ExtElt->getOperand(1);
36188 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
36190 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
36210 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
36213 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
36215 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
36252 SmallVector<SDValue, 4> ExtOps;
36253 for (SDValue Op : Vec->ops())
36265 static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
36274 SDValue Rdx =
36279 SDValue Index = ExtElt->getOperand(1);
36326 SDValue Lo = extractSubVector(Rdx, 0, DAG, DL, HalfSize);
36327 SDValue Hi = extractSubVector(Rdx, HalfElts, DAG, DL, HalfSize);
36333 SDValue Hi = DAG.getVectorShuffle(
36357 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
36358 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
36378 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
36381 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
36384 SDValue InputVector = N->getOperand(0);
36385 SDValue EltIdx = N->getOperand(1);
36423 SDValue Scl = InputVector.getOperand(1);
36434 if (SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI))
36440 SDValue MMXSrc = InputVector.getOperand(0);
36450 SDValue MMXSrc = InputVector.getOperand(0);
36460 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
36464 if (SDValue Cmp = combineHorizontalPredicateResult(N, DAG, Subtarget))
36468 if (SDValue MinMax = combineHorizontalMinMaxResult(N, DAG, Subtarget))
36471 if (SDValue V = combineReductionToHorizontal(N, DAG, Subtarget))
36474 if (SDValue V = scalarizeExtEltFP(N, DAG))
36494 if (SDValue BC =
36500 SDValue Mask = DAG.getConstant(MaskBit, dl, BCVT);
36501 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
36516 static SDValue
36520 SDValue Cond = N->getOperand(0);
36521 SDValue LHS = N->getOperand(1);
36522 SDValue RHS = N->getOperand(2);
36552 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
36577 SDValue CC = Cond.getOperand(2);
36602 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
36603 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
36609 SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
36610 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
36617 SDValue CastCond = DAG.getBitcast(AndNVT, Cond);
36618 SDValue CastRHS = DAG.getBitcast(AndNVT, RHS);
36619 SDValue AndN = DAG.getNode(X86ISD::ANDNP, DL, AndNVT, CastCond, CastRHS);
36630 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG,
36642 SDValue Cond = N->getOperand(0);
36643 SDValue TVal = N->getOperand(1);
36644 SDValue FVal = N->getOperand(2);
36645 SmallVector<SDValue, 4> CatOpsT, CatOpsF;
36659 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG) {
36660 SDValue Cond = N->getOperand(0);
36661 SDValue LHS = N->getOperand(1);
36662 SDValue RHS = N->getOperand(2);
36706 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
36727 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
36730 SDValue Cond = N->getOperand(0);
36795 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
36805 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
36809 SDValue Cond = N->getOperand(0);
36810 SDValue LHS = N->getOperand(1);
36811 SDValue RHS = N->getOperand(2);
36815 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
36992 SDValue AndNode = Cond.getOperand(0);
37049 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
37054 if (SDValue V = combineSelectOfTwoConstants(N, DAG))
37100 SDValue Other;
37110 SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
37111 SDValue CondRHS = Cond->getOperand(1);
37167 SDValue CondLHS = Cond->getOperand(0);
37168 SDValue CondRHS = Cond->getOperand(1);
37172 SDValue Other;
37181 SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
37215 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget))
37218 if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget))
37221 if (SDValue V = narrowVectorSelect(N, DAG, Subtarget))
37226 if (SDValue CondNot = IsNOT(Cond, DAG))
37234 SDValue newSelect = DAG.getNode(ISD::SELECT, DL, MVT::i64, Cond, LHS, RHS);
37247 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
37247 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
37273 SDValue CmpLHS = Cmp.getOperand(0);
37274 SDValue CmpRHS = Cmp.getOperand(1);
37283 SDValue OpRHS = CmpLHS.getOperand(2);
37304 auto AtomicSub = DAG.getAtomic(
37309 auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
37332 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
37354 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
37354 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
37366 SDValue Op1 = Cmp.getOperand(0);
37367 SDValue Op2 = Cmp.getOperand(1);
37369 SDValue SetCC;
37433 SDValue Op = SetCC.getOperand(0);
37472 static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
37473 X86::CondCode &CC1, SDValue &Flags,
37484 SDValue SetCC0, SetCC1;
37513 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
37513 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
37516 SDValue Carry = EFLAGS.getOperand(0);
37528 SDValue CarryOp1 = Carry.getOperand(1);
37542 SDValue SubCommute =
37564 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
37564 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
37568 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
37571 if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
37577 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
37582 SDValue FalseOp = N->getOperand(0);
37583 SDValue TrueOp = N->getOperand(1);
37585 SDValue Cond = N->getOperand(3);
37593 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
37595 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
37716 SDValue Ops[] = {FalseOp, Cond.getOperand(0),
37741 SDValue Flags;
37751 SDValue LOps[] = {FalseOp, TrueOp,
37753 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
37754 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
37756 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
37767 SDValue Add = TrueOp;
37768 SDValue Const = FalseOp;
37786 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
37787 SDValue CMov =
37809 SDValue Opd = N->getOperand(i);
37862 static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
37882 SDValue N0 = N->getOperand(0);
37883 SDValue N1 = N->getOperand(1);
37892 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
37893 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
37897 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
37905 SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
37916 SDValue ResLo =
37924 SDValue ResHi =
37930 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
37934 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
37944 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
38005 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
38007 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
38018 static SDValue combineMulToPMADDWD(SDNode *N, SelectionDAG &DAG,
38038 SDValue N0 = N->getOperand(0);
38039 SDValue N1 = N->getOperand(1);
38066 static SDValue combineMulToPMULDQ(SDNode *N, SelectionDAG &DAG,
38079 SDValue N0 = N->getOperand(0);
38080 SDValue N1 = N->getOperand(1);
38110 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
38115 if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget))
38118 if (SDValue V = combineMulToPMULDQ(N, DAG, Subtarget))
38148 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
38170 SDValue NewMul;
38252 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
38253 SDValue N0 = N->getOperand(0);
38254 SDValue N1 = N->getOperand(1);
38263 SDValue N00 = N0.getOperand(0);
38311 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG) {
38312 SDValue N0 = N->getOperand(0);
38313 SDValue N1 = N->getOperand(1);
38333 SDValue N00 = N0.getOperand(0);
38334 SDValue N01 = N0.getOperand(1);
38349 SDValue NN =
38364 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
38366 SDValue N0 = N->getOperand(0);
38367 SDValue N1 = N->getOperand(1);
38406 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
38407 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
38413 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
38421 SDValue N0 = N->getOperand(0);
38422 SDValue N1 = N->getOperand(1);
38499 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
38506 SDValue Op(N, 0);
38507 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
38513 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
38520 SDValue N0 = N->getOperand(0);
38521 SDValue N1 = N->getOperand(1);
38546 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
38555 SDValue N0 = N->getOperand(0);
38556 SDValue N1 = N->getOperand(1);
38593 SDValue Op(N, 0);
38594 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
38624 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
38639 SDValue Op(N, 0);
38640 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
38649 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
38657 SDValue N0 = N->getOperand(0);
38658 SDValue N1 = N->getOperand(1);
38659 SDValue CMP0 = N0.getOperand(1);
38660 SDValue CMP1 = N1.getOperand(1);
38667 SDValue CMP00 = CMP0->getOperand(0);
38668 SDValue CMP01 = CMP0->getOperand(1);
38706 SDValue FSetCC =
38711 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
38717 SDValue OnesOrZeroesF =
38730 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
38732 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
38738 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
38739 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
38741 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
38752 static SDValue combineANDXORWithAllOnesIntoANDNP(SDNode *N, SelectionDAG &DAG) {
38759 SDValue X, Y;
38760 SDValue N0 = N->getOperand(0);
38761 SDValue N1 = N->getOperand(1);
38763 if (SDValue Not = IsNOT(N0, DAG)) {
38766 } else if (SDValue Not = IsNOT(N1, DAG)) {
38783 static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
38792 SDValue Narrow = N->getOperand(0);
38800 SDValue N0 = Narrow->getOperand(0);
38801 SDValue N1 = Narrow->getOperand(1);
38832 SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, VT, N0, N1);
38849 static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
38852 SDValue N0 = N->getOperand(0);
38853 SDValue N1 = N->getOperand(1);
38859 SDValue N00 = N0.getOperand(0);
38860 SDValue N10 = N1.getOperand(0);
38878 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
38885 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
38887 SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
38888 SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
38913 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
38914 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
38920 static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
38924 SDValue Base = Ld->getBasePtr();
38929 SDValue ShiftedIndex = Base.getOperand(0);
38959 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
38970 SDValue N = Node->getOperand(i);
39014 SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0);
39015 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
39018 SDValue Index = getIndexFromUnindexedLoad(Ld);
39023 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
39026 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
39027 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
39039 static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
39052 SDValue N0 = N->getOperand(0);
39053 SDValue N1 = N->getOperand(1);
39064 SDValue X = N0.getOperand(0);
39069 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
39072 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
39076 SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32,
39084 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, VT, X,
39091 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
39094 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
39096 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
39099 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
39110 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
39130 SDValue Src = N->getOperand(0).getOperand(0);
39141 SDValue SubVec = Src.getOperand(0);
39161 SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
39164 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
39169 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
39189 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(0));
39190 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(1));
39197 if (SDValue V = combineParity(N, DAG, Subtarget))
39203 SmallVector<SDValue, 2> SrcOps;
39209 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
39218 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
39224 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
39227 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
39230 if (SDValue R = combineANDXORWithAllOnesIntoANDNP(N, DAG))
39233 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
39236 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
39241 SDValue Op(N, 0);
39242 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
39250 SDValue BitMask = N->getOperand(1);
39251 SDValue SrcVec = N->getOperand(0).getOperand(0);
39277 if (SDValue Shuffle = combineX86ShufflesRecursively(
39289 static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
39297 SDValue N0 = peekThroughBitcasts(N->getOperand(0));
39298 SDValue N1 = peekThroughBitcasts(N->getOperand(1));
39329 SDValue X = N->getOperand(0);
39330 SDValue Y =
39337 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
39337 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
39337 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
39341 SDValue N0 = N->getOperand(0);
39342 SDValue N1 = N->getOperand(1);
39384 static SDValue combineLogicBlendIntoConditionalNegate(
39385 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
39385 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
39385 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
39402 SDValue V;
39410 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
39411 SDValue SubOp2 = Mask;
39426 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
39438 static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
39447 SDValue X, Y, Mask;
39466 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
39489 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, EVT ExtTy,
39489 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, EVT ExtTy,
39491 SDValue Cmp = Op.getOperand(1);
39495 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
39498 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
39499 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
39511 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
39538 SDValue LHS = OR->getOperand(0);
39539 SDValue RHS = OR->getOperand(1);
39562 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, VT, DAG);
39563 SDValue Ret, NewRHS;
39590 static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
39593 SDValue N0 = N->getOperand(0);
39594 SDValue N1 = N->getOperand(1);
39608 SmallVector<SDValue, 2> SrcOps;
39614 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
39626 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
39629 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
39632 if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
39635 if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
39640 SDValue Op(N, 0);
39641 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
39667 SDValue ShAmt0 = N0.getOperand(1);
39670 SDValue ShAmt1 = N1.getOperand(1);
39675 SDValue ShMsk0;
39682 SDValue ShMsk1;
39697 SDValue Op0 = N0.getOperand(0);
39698 SDValue Op1 = N1.getOperand(0);
39721 SDValue Sum = ShAmt1.getOperand(0);
39723 SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
39742 SDValue Mask = ShAmt1.getOperand(1);
39745 SDValue ShAmt1Op0 = ShAmt1.getOperand(0);
39771 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
39777 SDValue N0 = N->getOperand(0);
39778 SDValue N1 = N->getOperand(1);
39789 SDValue Shift = N0.getOperand(0);
39807 SDValue ShiftOp = Shift.getOperand(0);
39812 SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
39826 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
39846 SDValue Shift = N->getOperand(0);
39847 SDValue Ones = N->getOperand(1);
39880 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
39880 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
39897 if (SDValue UMin = MatchMinMax(In, ISD::UMIN, C2))
39903 if (SDValue SMin = MatchMinMax(In, ISD::SMIN, C2))
39908 if (SDValue SMax = MatchMinMax(In, ISD::SMAX, C1))
39909 if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, C2))
39927 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
39927 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
39950 if (SDValue SMin = MatchMinMax(In, ISD::SMIN, SignedMax))
39951 if (SDValue SMax = MatchMinMax(SMin, ISD::SMAX, SignedMin))
39954 if (SDValue SMax = MatchMinMax(In, ISD::SMAX, SignedMin))
39955 if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, SignedMax))
39961 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
39961 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
39977 if (auto USatVal = detectSSatPattern(In, VT, true)) {
39979 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
40002 if (auto USatVal = detectSSatPattern(In, VT, true)) {
40009 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
40012 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
40020 if (auto SSatVal = detectSSatPattern(In, VT))
40029 SDValue SatVal;
40030 if (auto SSatVal = detectSSatPattern(In, VT)) {
40033 } else if (auto USatVal = detectUSatPattern(In, VT, DAG, DL)) {
40044 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
40054 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
40066 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,
40066 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,
40107 for (SDValue Op : V->ops()) {
40119 auto LHS = In.getOperand(0);
40120 auto RHS = In.getOperand(1);
40127 SDValue Operands[3];
40143 SDValue VecOnes = DAG.getConstant(1, DL, InVT);
40170 SDValue Op0, Op1;
40202 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
40228 SDValue Ptr1 = Ld->getBasePtr();
40229 SDValue Ptr2 = DAG.getMemBasePlusOffset(Ptr1, HalfAlign, dl);
40232 SDValue Load1 =
40235 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
40239 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
40242 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
40253 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
40256 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
40267 static int getOneTrueElt(SDValue V) {
40282 const SDValue &Op = BV->getOperand(i);
40303 SelectionDAG &DAG, SDValue &Addr,
40304 SDValue &Index, unsigned &Alignment) {
40327 static SDValue
40334 SDValue Addr, VecIndex;
40344 SDValue Load =
40349 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
40354 static SDValue
40371 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
40373 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
40392 SDValue NewML = DAG.getMaskedLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
40396 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
40402 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
40412 if (SDValue ScalarLoad = reduceMaskedLoadToScalarLoad(Mld, DAG, DCI))
40416 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
40427 static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
40433 SDValue Addr, VecIndex;
40442 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
40450 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
40464 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG))
40469 SDValue Mask = Mst->getMask();
40476 SDValue Value = Mst->getValue();
40488 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
40495 SDValue StoredVal = St->getValue();
40525 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT));
40539 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
40542 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
40546 SDValue Ptr0 = St->getBasePtr();
40547 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, 4, dl);
40549 SDValue Ch0 =
40552 SDValue Ch1 =
40608 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32, St->getValue());
40632 if (SDValue Avg = detectAVGPattern(St->getValue(), St->getMemoryVT(), DAG,
40639 if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
40643 if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
40671 SmallVector<SDValue, 8> Ops;
40689 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(),
40699 SDValue LoAddr = Ld->getBasePtr();
40700 SDValue HiAddr = DAG.getMemBasePlusOffset(LoAddr, 4, LdDL);
40702 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
40705 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr,
40716 SDValue LoSt =
40719 SDValue HiSt = DAG.getStore(St->getChain(), StDL, HiLd, HiAddr,
40734 SDValue OldExtract = St->getOperand(1);
40735 SDValue ExtOp0 = OldExtract.getOperand(0);
40738 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
40739 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
40762 static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
40762 static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
40803 SmallVector<SDValue, 2> SrcOps;
40805 SDValue BC = peekThroughBitcasts(Op);
40830 SDValue A, B;
40836 SDValue C, D;
40913 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
40916 SDValue LHS = N->getOperand(0);
40917 SDValue RHS = N->getOperand(1);
40936 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
40940 SDValue Src = N->getOperand(0);
40967 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
40968 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
40988 SDValue Op0 = Src.getOperand(0);
40989 SDValue Op1 = Src.getOperand(1);
41005 SDValue Op0 = Src.getOperand(0);
41006 SDValue Op1 = Src.getOperand(1);
41015 SDValue Op0 = Src.getOperand(0);
41016 SDValue Op1 = Src.getOperand(1);
41031 static SDValue combineVectorTruncationWithPACKUS(SDNode *N, const SDLoc &DL,
41034 SDValue In = N->getOperand(0);
41045 static SDValue combineVectorTruncationWithPACKSS(SDNode *N, const SDLoc &DL,
41048 SDValue In = N->getOperand(0);
41061 static SDValue combineVectorTruncation(SDNode *N, SelectionDAG &DAG,
41067 SDValue In = N->getOperand(0);
41108 static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL,
41118 SDValue In = N->getOperand(0);
41169 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
41169 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
41195 SDValue LHS = Src.getOperand(0).getOperand(0);
41196 SDValue RHS = Src.getOperand(0).getOperand(1);
41223 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
41223 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
41234 SDValue SSatVal = detectSSatPattern(In, VT);
41240 SDValue N0 = SSatVal.getOperand(0);
41241 SDValue N1 = SSatVal.getOperand(1);
41246 SDValue N00 = N0.getOperand(0);
41247 SDValue N01 = N0.getOperand(1);
41248 SDValue N10 = N1.getOperand(0);
41249 SDValue N11 = N1.getOperand(1);
41293 SDValue ZExtIn, SExtIn;
41295 SDValue N00Elt = N00.getOperand(i);
41296 SDValue N01Elt = N01.getOperand(i);
41297 SDValue N10Elt = N10.getOperand(i);
41298 SDValue N11Elt = N11.getOperand(i);
41324 SDValue N00In = N00Elt.getOperand(0);
41325 SDValue N01In = N01Elt.getOperand(0);
41326 SDValue N10In = N10Elt.getOperand(0);
41327 SDValue N11In = N11Elt.getOperand(0);
41354 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
41357 SDValue Src = N->getOperand(0);
41361 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
41365 if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))
41369 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
41373 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
41377 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
41383 SDValue BCSrc = Src.getOperand(0);
41389 if (SDValue V = combineVectorSignBitsTruncation(N, DL, DAG, Subtarget))
41395 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG) {
41397 SDValue In = N->getOperand(0);
41400 if (auto SSatVal = detectSSatPattern(In, VT))
41402 if (auto USatVal = detectUSatPattern(In, VT, DAG, DL))
41417 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
41427 SDValue Op = peekThroughBitcasts(SDValue(N, 0));
41438 if (SDValue NegOp0 = isFNEG(DAG, SVOp->getOperand(0).getNode(), Depth + 1))
41448 SDValue InsVector = Op.getOperand(0);
41449 SDValue InsVal = Op.getOperand(1);
41452 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
41462 SDValue Op1 = Op.getOperand(1);
41463 SDValue Op0 = Op.getOperand(0);
41541 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
41544 SDValue Arg = isFNEG(DAG, N);
41561 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
41562 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
41591 char X86TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
41634 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
41634 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
41639 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth))
41660 SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
41686 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
41699 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
41700 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
41709 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
41715 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
41719 SDValue LHS = N->getOperand(0);
41730 static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
41742 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
41748 if (SDValue SetCC = foldXor1SetCC(N, DAG))
41751 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
41754 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
41760 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
41763 SDValue Op0 = N->getOperand(0);
41764 SDValue Op1 = N->getOperand(1);
41789 static bool isNullFPScalarOrVectorConst(SDValue V) {
41799 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
41799 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
41810 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
41812 SDValue N0 = N->getOperand(0);
41813 SDValue N1 = N->getOperand(1);
41842 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
41845 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
41849 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
41852 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
41859 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
41866 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
41873 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
41885 if (SDValue NewVal = combineFneg(N, DAG, Subtarget))
41892 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
41912 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
41925 SDValue Op0 = N->getOperand(0);
41926 SDValue Op1 = N->getOperand(1);
41969 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
41970 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
41977 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
41989 SDValue In = N->getOperand(0);
42002 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
42003 SDValue VZLoad =
42008 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
42019 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
42024 SDValue In = N->getOperand(0);
42037 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
42038 SDValue VZLoad =
42043 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
42055 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
42069 if (SDValue Not = IsNOT(N->getOperand(0), DAG))
42075 SDValue Op(N, 0);
42076 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42083 static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
42085 SDValue N0 = N->getOperand(0);
42086 SDValue N1 = N->getOperand(1);
42091 if (SDValue DemandedN1 = DAG.GetDemandedBits(N1, DemandedMask))
42098 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
42103 SDValue N0 = N->getOperand(0);
42104 SDValue N1 = N->getOperand(1);
42111 SDValue IntermediateBitwidthOp;
42122 SDValue CMovOp0 = N0.getOperand(0);
42123 SDValue CMovOp1 = N0.getOperand(1);
42150 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
42159 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
42163 if (SDValue V = combineSextInRegCmov(N, DAG))
42167 SDValue N0 = N->getOperand(0);
42168 SDValue N1 = N->getOperand(1);
42179 SDValue N00 = N0.getOperand(0);
42188 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32,
42201 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
42212 SDValue Add = Ext->getOperand(0);
42249 SDValue AddOp0 = Add.getOperand(0);
42250 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
42251 SDValue NewConstant = DAG.getConstant(AddConstant, SDLoc(Add), VT);
42273 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
42274 SDValue CMovN = Extend->getOperand(0);
42283 SDValue CMovOp0 = CMovN.getOperand(0);
42284 SDValue CMovOp1 = CMovN.getOperand(1);
42308 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
42320 static SDValue
42333 SDValue N0 = N->getOperand(0);
42348 SDValue N00 = N0.getOperand(0);
42354 SDValue Vec;
42378 SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
42385 SmallVector<SDValue, 32> Bits;
42391 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
42409 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
42411 SDValue N0 = N->getOperand(0);
42442 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
42450 static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
42453 SDValue N0 = N->getOperand(0);
42458 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
42464 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
42473 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
42477 if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
42481 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
42484 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
42490 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
42505 SDValue A = N->getOperand(0);
42506 SDValue B = N->getOperand(1);
42507 SDValue C = N->getOperand(2);
42520 SDValue Vec = V.getOperand(0);
42522 SDValue NegVal =
42552 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
42560 SDValue N2 = N->getOperand(2);
42564 SDValue NegN2 = TLI.getNegatedExpression(N2, DAG, LegalOperations, CodeSize);
42574 static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
42582 SDValue N0 = N->getOperand(0);
42588 SDValue N00 = N0.getOperand(0);
42602 SDValue N00 = N0.getOperand(0);
42611 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
42615 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
42618 if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
42622 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
42625 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
42628 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
42634 SDValue N00 = N0.getOperand(0);
42635 SDValue N01 = N0.getOperand(1);
42649 static SDValue combineVectorSizedSetCCEquality(SDNode *SetCC, SelectionDAG &DAG,
42655 SDValue X = SetCC->getOperand(0);
42656 SDValue Y = SetCC->getOperand(1);
42728 SDValue OrigX = X.getOperand(0);
42752 SDValue Cmp;
42758 SDValue A = ScalarToVector(X.getOperand(0).getOperand(0));
42759 SDValue B = ScalarToVector(X.getOperand(0).getOperand(1));
42760 SDValue C = ScalarToVector(X.getOperand(1).getOperand(0));
42761 SDValue D = ScalarToVector(X.getOperand(1).getOperand(1));
42763 SDValue Cmp1 = DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
42764 SDValue Cmp2 = DAG.getSetCC(DL, CmpVT, C, D, ISD::SETNE);
42767 SDValue Cmp1 = DAG.getNode(ISD::XOR, DL, VecVT, A, B);
42768 SDValue Cmp2 = DAG.getNode(ISD::XOR, DL, VecVT, C, D);
42771 SDValue Cmp1 = DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
42772 SDValue Cmp2 = DAG.getSetCC(DL, CmpVT, C, D, ISD::SETEQ);
42776 SDValue VecX = ScalarToVector(X);
42777 SDValue VecY = ScalarToVector(Y);
42794 SDValue BCCmp = DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64,
42796 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
42798 SDValue SetCC = getSETCC(X86CC, PT, DL, DAG);
42806 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
42807 SDValue FFFFs = DAG.getConstant(OpSize == 128 ? 0xFFFF : 0xFFFFFFFF, DL,
42815 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
42818 SDValue LHS = N->getOperand(0);
42819 SDValue RHS = N->getOperand(1);
42829 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, RHS, LHS.getOperand(1));
42836 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1));
42840 if (SDValue V = combineVectorSizedSetCCEquality(N, DAG, Subtarget))
42882 SDValue Setcc = DAG.getNode(ISD::SETCC, DL, OpVT, LHS, RHS,
42896 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
42899 SDValue Src = N->getOperand(0);
42925 if (SDValue NotSrc = IsNOT(Src, DAG)) {
42943 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
42946 SDValue Mask = cast<X86MaskedGatherScatterSDNode>(N)->getMask();
42957 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
42961 SDValue Chain = GorS->getChain();
42962 SDValue Index = GorS->getIndex();
42963 SDValue Mask = GorS->getMask();
42964 SDValue Base = GorS->getBasePtr();
42965 SDValue Scale = GorS->getScale();
42984 SDValue Ops[] = { Chain, Gather->getPassThru(),
42992 SDValue Ops[] = { Chain, Scatter->getValue(),
43013 SDValue Ops[] = { Chain, Gather->getPassThru(),
43021 SDValue Ops[] = { Chain, Scatter->getValue(),
43040 SDValue Ops[] = { Chain, Gather->getPassThru(),
43048 SDValue Ops[] = { Chain, Scatter->getValue(),
43069 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
43073 SDValue EFLAGS = N->getOperand(1);
43076 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
43083 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
43086 SDValue EFLAGS = N->getOperand(3);
43092 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
43093 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
43101 static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
43134 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
43136 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
43137 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
43139 SDValue Res = DAG.getBitcast(VT, NewAnd);
43149 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
43153 SDValue Trunc = N->getOperand(0);
43157 SDValue ExtElt = Trunc.getOperand(0);
43174 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
43176 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
43181 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
43183 SDValue Op0 = N->getOperand(0);
43194 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
43209 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
43214 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
43218 SDValue Op0 = N->getOperand(0);
43229 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
43246 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
43252 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
43253 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
43278 SDValue FILDChain = Subtarget.getTargetLowering()->BuildFILD(
43285 if (SDValue V = combineToFPTruncExtElt(N, DAG))
43291 static bool needCarryOrOverflowFlag(SDValue Flags) {
43329 static bool onlyZeroFlagUsed(SDValue Flags) {
43355 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG) {
43365 SDValue Op = N->getOperand(0);
43427 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
43428 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
43443 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
43449 SDValue LHS = N->getOperand(0);
43450 SDValue RHS = N->getOperand(1);
43456 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
43462 SDValue Ops[] = {N0, N1};
43465 SDValue Op(N, 0);
43477 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
43478 if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
43488 SDValue Op0 = N->getOperand(0);
43489 SDValue Op1 = N->getOperand(1);
43499 static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
43511 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
43512 SDValue Res1 =
43521 if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
43535 static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
43537 SDValue X = N->getOperand(0);
43538 SDValue Y = N->getOperand(1);
43583 SDValue EFLAGS = Y->getOperand(1);
43590 SDValue NewSub = DAG.getNode(
43593 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
43610 SDValue EFLAGS = Y->getOperand(1);
43620 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
43623 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
43633 SDValue Cmp = Y.getOperand(1);
43639 SDValue Z = Cmp.getOperand(0);
43651 SDValue Zero = DAG.getConstant(0, DL, ZVT);
43653 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
43665 SDValue One = DAG.getConstant(1, DL, ZVT);
43666 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
43673 SDValue One = DAG.getConstant(1, DL, ZVT);
43674 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
43691 static SDValue combineLoopMAddPattern(SDNode *N, SelectionDAG &DAG,
43703 SDValue Op0 = N->getOperand(0);
43704 SDValue Op1 = N->getOperand(1);
43715 SDValue MulOp, OtherOp;
43732 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, MulOp->getOperand(0));
43733 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, MulOp->getOperand(1));
43741 SDValue Madd = SplitOpsAndApply(DAG, Subtarget, DL, MAddVT, { N0, N1 },
43744 SDValue Zero = DAG.getConstant(0, DL, Madd.getSimpleValueType());
43745 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Madd, Zero);
43754 static SDValue combineLoopSADPattern(SDNode *N, SelectionDAG &DAG,
43782 SDValue AbsOp = N->getOperand(0);
43783 SDValue OtherOp = N->getOperand(1);
43790 SDValue SadOp0, SadOp1;
43798 SDValue Sad = createPSADBW(DAG, SadOp0, SadOp1, DL, Subtarget);
43814 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, DL, ResVT));
43829 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
43829 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
43829 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
43866 SDValue Mul;
43868 SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
43941 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
43941 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
43941 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
43955 SDValue N00 = N0.getOperand(0);
43956 SDValue N01 = N0.getOperand(1);
43957 SDValue N10 = N1.getOperand(0);
43958 SDValue N11 = N1.getOperand(1);
43993 SDValue In0, In1;
43995 SDValue N00Elt = N00.getOperand(i);
43996 SDValue N01Elt = N01.getOperand(i);
43997 SDValue N10Elt = N10.getOperand(i);
43998 SDValue N11Elt = N11.getOperand(i);
44024 SDValue N00In = N00Elt.getOperand(0);
44025 SDValue N01In = N01Elt.getOperand(0);
44026 SDValue N10In = N10Elt.getOperand(0);
44027 SDValue N11In = N11Elt.getOperand(0);
44059 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
44064 if (SDValue Sad = combineLoopSADPattern(N, DAG, Subtarget))
44066 if (SDValue MAdd = combineLoopMAddPattern(N, DAG, Subtarget))
44070 SDValue Op0 = N->getOperand(0);
44071 SDValue Op1 = N->getOperand(1);
44073 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))
44075 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))
44102 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
44110 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
44118 static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG,
44120 SDValue Op0 = N->getOperand(0);
44121 SDValue Op1 = N->getOperand(1);
44135 SDValue SubusLHS, SubusRHS;
44141 SDValue MaxLHS = Op0.getOperand(0);
44142 SDValue MaxRHS = Op0.getOperand(1);
44151 SDValue MinLHS = Op1.getOperand(0);
44152 SDValue MinRHS = Op1.getOperand(1);
44188 SDValue SaturationConst =
44192 SDValue UMin = DAG.getNode(ISD::UMIN, SDLoc(SubusLHS), ExtType, SubusRHS,
44194 SDValue NewSubusLHS =
44196 SDValue NewSubusRHS = DAG.getZExtOrTrunc(UMin, SDLoc(SubusRHS), ShrinkedType);
44197 SDValue Psubus = DAG.getNode(ISD::USUBSAT, SDLoc(N), ShrinkedType,
44205 static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
44208 SDValue Op0 = N->getOperand(0);
44209 SDValue Op1 = N->getOperand(1);
44221 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
44244 if (SDValue V = combineSubToSubus(N, DAG, Subtarget))
44250 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
44268 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
44269 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
44282 SDValue Op0 = Ops[0];
44292 if (SDValue Ld =
44337 SmallVector<SDValue, 2> Src;
44349 SmallVector<SDValue, 2> Src;
44352 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f32, Src);
44360 SmallVector<SDValue, 2> LHS, RHS;
44379 static SDValue combineConcatVectors(SDNode *N, SelectionDAG &DAG,
44391 SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
44392 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
44400 static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
44411 SDValue Vec = N->getOperand(0);
44412 SDValue SubVec = N->getOperand(1);
44444 SDValue Ins = SubVec.getOperand(0);
44480 SmallVector<SDValue, 2> SubVectorOps;
44482 if (SDValue Fold =
44508 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
44509 SDValue BcastLd =
44526 static SDValue narrowExtractedVectorSelect(SDNode *Ext, SelectionDAG &DAG) {
44527 SDValue Sel = peekThroughBitcasts(Ext->getOperand(0));
44528 SmallVector<SDValue, 4> CatOps;
44571 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
44572 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
44573 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
44574 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
44578 static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
44595 SDValue InVec = N->getOperand(0);
44596 SDValue InVecBC = peekThroughBitcasts(InVec);
44608 SDValue NotOp = V->getOperand(0);
44614 SDValue Concat = split256IntArith(InVecBC, DAG);
44623 if (SDValue V = narrowExtractedVectorSelect(N, DAG))
44658 SDValue NewIndex = DAG.getIntPtrConstant(IndexValScaled, DL);
44659 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT,
44692 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
44693 SDValue BcastLd =
44739 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
44740 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
44741 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
44749 static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
44751 SDValue Src = N->getOperand(0);
44786 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
44789 SDValue LHS = N->getOperand(0);
44790 SDValue RHS = N->getOperand(1);
44838 static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
44842 SDValue In = N->getOperand(0);
44855 SDValue Load =
44868 SDValue Op(N, 0);
44870 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
44877 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
44891 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
45085 SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc& dl,
45086 SDValue Value, SDValue Addr,
45086 SDValue Value, SDValue Addr,
45101 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
45147 SDValue N0 = Op.getOperand(0);
45161 SDValue N0 = Op.getOperand(0);
45162 SDValue N1 = Op.getOperand(1);
45586 SDValue X86TargetLowering::LowerAsmOutputForConstraint(
45587 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
45587 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
45604 SDValue CC = getSETCC(Cond, Flag, DL, DAG);
45606 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
45613 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
45615 std::vector<SDValue>&Ops,
45617 SDValue Result;
lib/Target/X86/X86ISelLowering.h 678 bool isZeroNode(SDValue Elt);
692 bool isConstantSplat(SDValue Op, APInt &SplatVal);
718 SDValue getPICJumpTableRelocBase(SDValue Table,
718 SDValue getPICJumpTableRelocBase(SDValue Table,
762 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
762 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
769 SmallVectorImpl<SDValue> &Results,
775 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
778 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
803 bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
808 char isNegatibleForFree(SDValue Op, SelectionDAG &DAG, bool LegalOperations,
812 SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
812 SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
860 bool hasAndNotCompare(SDValue Y) const override;
862 bool hasAndNot(SDValue Y) const override;
864 bool hasBitTest(SDValue X, SDValue Y) const override;
864 bool hasBitTest(SDValue X, SDValue Y) const override;
867 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
867 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
874 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;
909 bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
914 void computeKnownBitsForTargetNode(const SDValue Op,
921 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
926 bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op,
933 bool SimplifyDemandedBitsForTargetNode(SDValue Op,
940 SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
941 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
946 SDValue unwrapAddress(SDValue N) const override;
946 SDValue unwrapAddress(SDValue N) const override;
948 SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
965 void LowerAsmOperandForConstraint(SDValue Op,
967 std::vector<SDValue> &Ops,
984 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, SDLoc DL,
984 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, SDLoc DL,
984 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, SDLoc DL,
1050 bool isZExtFree(SDValue Val, EVT VT2) const override;
1054 bool isVectorLoadExtDesirable(SDValue) const override;
1126 SDValue C) const override;
1139 bool shouldScalarizeBinop(SDValue) const override;
1197 SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
1197 SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
1206 SDValue BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, SDValue StackSlot,
1206 SDValue BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, SDValue StackSlot,
1206 SDValue BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, SDValue StackSlot,
1206 SDValue BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, SDValue StackSlot,
1249 SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value,
1249 SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value,
1250 SDValue Addr, SelectionDAG &DAG)
1278 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1278 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1278 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1282 SmallVectorImpl<SDValue> &InVals,
1284 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1284 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1289 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1289 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1289 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1289 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1298 bool IsEligibleForTailCallOptimization(SDValue Callee,
1305 const SmallVectorImpl<SDValue> &OutVals,
1308 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
1308 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
1309 SDValue Chain, bool IsTailCall,
1318 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool isSigned) const;
1318 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool isSigned) const;
1320 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1320 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1321 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
1321 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
1322 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1322 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1323 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1323 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1327 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1327 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1328 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1328 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1329 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1329 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1330 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1330 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1331 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
1331 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
1335 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1335 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1338 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1338 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1339 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1339 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1340 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1340 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1341 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1341 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1342 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1342 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1343 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1343 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1344 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1344 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1345 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
1345 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
1346 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1346 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1347 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1347 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1348 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1348 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1349 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1349 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1350 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1350 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1351 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1351 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1352 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1352 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1353 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1353 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1354 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
1354 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
1355 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1355 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1356 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1356 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1357 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
1357 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
1358 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1358 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1359 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
1359 SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
1360 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
1360 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
1361 SDValue LowerGC_TRANSITION_START(SDValue Op, SelectionDAG &DAG) const;
1361 SDValue LowerGC_TRANSITION_START(SDValue Op, SelectionDAG &DAG) const;
1362 SDValue LowerGC_TRANSITION_END(SDValue Op, SelectionDAG &DAG) const;
1362 SDValue LowerGC_TRANSITION_END(SDValue Op, SelectionDAG &DAG) const;
1363 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1363 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1364 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
1364 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
1365 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1365 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1366 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
1366 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
1368 SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
1368 SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
1371 SDValue
1372 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1375 SmallVectorImpl<SDValue> &InVals) const override;
1376 SDValue LowerCall(CallLoweringInfo &CLI,
1377 SmallVectorImpl<SDValue> &InVals) const override;
1379 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1379 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1381 const SmallVectorImpl<SDValue> &OutVals,
1393 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
1482 SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl,
1482 SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl,
1482 SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl,
1486 SDValue ConvertCmpIfNecessary(SDValue Cmp, SelectionDAG &DAG) const;
1486 SDValue ConvertCmpIfNecessary(SDValue Cmp, SelectionDAG &DAG) const;
1490 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1,
1490 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1,
1490 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1,
1493 SDValue &X86CC) const;
1496 bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override;
1499 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1499 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1504 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1504 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1510 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1526 const SDValue &getValue() const { return getOperand(1); }
1527 const SDValue &getBasePtr() const { return getOperand(2); }
1545 const SDValue &getValue() const { return getOperand(1); }
1546 const SDValue &getBasePtr() const { return getOperand(2); }
1547 const SDValue &getMask() const { return getOperand(3); }
1615 const SDValue &getBasePtr() const { return getOperand(3); }
1616 const SDValue &getIndex() const { return getOperand(4); }
1617 const SDValue &getMask() const { return getOperand(2); }
1618 const SDValue &getScale() const { return getOperand(5); }
1633 const SDValue &getPassThru() const { return getOperand(1); }
1647 const SDValue &getValue() const { return getOperand(1); }
lib/Target/X86/X86InstrInfo.cpp 5615 std::vector<SDValue> AddrOps;
5616 std::vector<SDValue> BeforeOps;
5617 std::vector<SDValue> AfterOps;
5621 SDValue Op = N->getOperand(i);
5629 SDValue Chain = N->getOperand(NumOps-1);
lib/Target/X86/X86SelectionDAGInfo.cpp 46 SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
47 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
47 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
47 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
48 SDValue Size, unsigned Align, bool isVolatile,
95 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
95 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
104 SDValue InFlag;
106 SDValue Count;
163 SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
186 static SDValue emitRepmovs(const X86Subtarget &Subtarget, SelectionDAG &DAG,
187 const SDLoc &dl, SDValue Chain, SDValue Dst,
187 const SDLoc &dl, SDValue Chain, SDValue Dst,
188 SDValue Src, SDValue Size, MVT AVT) {
188 SDValue Src, SDValue Size, MVT AVT) {
194 SDValue InFlag;
203 SDValue Ops[] = {Chain, DAG.getValueType(AVT), InFlag};
208 static SDValue emitRepmovsB(const X86Subtarget &Subtarget, SelectionDAG &DAG,
209 const SDLoc &dl, SDValue Chain, SDValue Dst,
209 const SDLoc &dl, SDValue Chain, SDValue Dst,
210 SDValue Src, uint64_t Size) {
236 static SDValue emitConstantSizeRepmov(
238 SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
238 SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
238 SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
261 SDValue RepMovs =
277 SmallVector<SDValue, 4> Results;
292 SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
293 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
293 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
293 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
294 SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
lib/Target/X86/X86SelectionDAGInfo.h 34 SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
35 SDValue Chain, SDValue Dst, SDValue Src,
35 SDValue Chain, SDValue Dst, SDValue Src,
35 SDValue Chain, SDValue Dst, SDValue Src,
36 SDValue Size, unsigned Align, bool isVolatile,
39 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
40 SDValue Chain, SDValue Dst, SDValue Src,
40 SDValue Chain, SDValue Dst, SDValue Src,
40 SDValue Chain, SDValue Dst, SDValue Src,
41 SDValue Size, unsigned Align, bool isVolatile,
lib/Target/XCore/XCoreISelDAGToDAG.cpp 48 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
64 bool SelectADDRspii(SDValue Addr, SDValue &Base, SDValue &Offset);
64 bool SelectADDRspii(SDValue Addr, SDValue &Base, SDValue &Offset);
64 bool SelectADDRspii(SDValue Addr, SDValue &Base, SDValue &Offset);
66 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
67 std::vector<SDValue> &OutOps) override;
86 bool XCoreDAGToDAGISel::SelectADDRspii(SDValue Addr, SDValue &Base,
86 bool XCoreDAGToDAGISel::SelectADDRspii(SDValue Addr, SDValue &Base,
87 SDValue &Offset) {
110 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
111 std::vector<SDValue> &OutOps) {
112 SDValue Reg;
140 SDValue MskSize = getI32Imm(32 - countLeadingZeros((uint32_t)Val), dl);
146 SDValue CPIdx = CurDAG->getTargetConstantPool(
162 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
169 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
176 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
183 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
190 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
197 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) };
215 static SDValue
216 replaceInChain(SelectionDAG *CurDAG, SDValue Chain, SDValue Old, SDValue New)
216 replaceInChain(SelectionDAG *CurDAG, SDValue Chain, SDValue Old, SDValue New)
216 replaceInChain(SelectionDAG *CurDAG, SDValue Chain, SDValue Old, SDValue New)
222 SmallVector<SDValue, 8> Ops;
240 SDValue Chain = N->getOperand(0);
241 SDValue Addr = N->getOperand(1);
247 SDValue nextAddr = Addr->getOperand(2);
248 SDValue CheckEventChainOut(Addr.getNode(), 1);
254 SDValue CheckEventChainIn = Addr->getOperand(0);
255 SDValue NewChain = replaceInChain(CurDAG, Chain, CheckEventChainOut,
265 SDValue constOne = getI32Imm(1, dl);
266 SDValue Glue =
lib/Target/XCore/XCoreISelLowering.cpp 178 bool XCoreTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
196 SDValue XCoreTargetLowering::
197 LowerOperation(SDValue Op, SelectionDAG &DAG) const {
231 SmallVectorImpl<SDValue>&Results,
247 SDValue XCoreTargetLowering::getGlobalAddressWrapper(SDValue GA,
247 SDValue XCoreTargetLowering::getGlobalAddressWrapper(SDValue GA,
277 SDValue XCoreTargetLowering::
278 LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const
287 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, FoldedOffset);
291 SDValue Remaining = DAG.getConstant(Offset - FoldedOffset, DL, MVT::i32);
303 SDValue CP = DAG.getConstantPool(GAI, MVT::i32);
309 SDValue XCoreTargetLowering::
310 LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
315 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT);
320 SDValue XCoreTargetLowering::
321 LowerConstantPool(SDValue Op, SelectionDAG &DAG) const
327 SDValue Res;
342 SDValue XCoreTargetLowering::
343 LowerBR_JT(SDValue Op, SelectionDAG &DAG) const
345 SDValue Chain = Op.getOperand(0);
346 SDValue Table = Op.getOperand(1);
347 SDValue Index = Op.getOperand(2);
353 SDValue TargetJT = DAG.getTargetJumpTable(JT->getIndex(), MVT::i32);
360 SDValue ScaledIndex = DAG.getNode(ISD::SHL, dl, MVT::i32, Index,
366 SDValue XCoreTargetLowering::lowerLoadWordFromAlignedBasePlusOffset(
367 const SDLoc &DL, SDValue Chain, SDValue Base, int64_t Offset,
367 const SDLoc &DL, SDValue Chain, SDValue Base, int64_t Offset,
376 SDValue LowAddr, HighAddr;
389 SDValue LowShift = DAG.getConstant((Offset - LowOffset) * 8, DL, MVT::i32);
390 SDValue HighShift = DAG.getConstant((HighOffset - Offset) * 8, DL, MVT::i32);
392 SDValue Low = DAG.getLoad(PtrVT, DL, Chain, LowAddr, MachinePointerInfo());
393 SDValue High = DAG.getLoad(PtrVT, DL, Chain, HighAddr, MachinePointerInfo());
394 SDValue LowShifted = DAG.getNode(ISD::SRL, DL, MVT::i32, Low, LowShift);
395 SDValue HighShifted = DAG.getNode(ISD::SHL, DL, MVT::i32, High, HighShift);
396 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i32, LowShifted, HighShifted);
399 SDValue Ops[] = { Result, Chain };
403 static bool isWordAligned(SDValue Value, SelectionDAG &DAG)
409 SDValue XCoreTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
409 SDValue XCoreTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
421 SDValue Chain = LD->getChain();
422 SDValue BasePtr = LD->getBasePtr();
430 SDValue NewBasePtr = BasePtr->getOperand(0);
437 SDValue NewBasePtr = DAG.getGlobalAddress(GV, DL,
445 SDValue Low =
449 SDValue HighAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
451 SDValue High =
455 SDValue HighShifted = DAG.getNode(ISD::SHL, DL, MVT::i32, High,
457 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i32, Low, HighShifted);
460 SDValue Ops[] = { Result, Chain };
480 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
480 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
481 SDValue Ops[] = { CallResult.first, CallResult.second };
485 SDValue XCoreTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
485 SDValue XCoreTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
495 SDValue Chain = ST->getChain();
496 SDValue BasePtr = ST->getBasePtr();
497 SDValue Value = ST->getValue();
501 SDValue Low = Value;
502 SDValue High = DAG.getNode(ISD::SRL, dl, MVT::i32, Value,
504 SDValue StoreLow = DAG.getTruncStore(
507 SDValue HighAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, BasePtr,
509 SDValue StoreHigh = DAG.getTruncStore(
534 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
534 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
538 SDValue XCoreTargetLowering::
539 LowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const
544 SDValue LHS = Op.getOperand(0);
545 SDValue RHS = Op.getOperand(1);
546 SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
547 SDValue Hi = DAG.getNode(XCoreISD::MACCS, dl,
550 SDValue Lo(Hi.getNode(), 1);
551 SDValue Ops[] = { Lo, Hi };
555 SDValue XCoreTargetLowering::
556 LowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const
561 SDValue LHS = Op.getOperand(0);
562 SDValue RHS = Op.getOperand(1);
563 SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
564 SDValue Hi = DAG.getNode(XCoreISD::LMUL, dl,
567 SDValue Lo(Hi.getNode(), 1);
568 SDValue Ops[] = { Lo, Hi };
578 isADDADDMUL(SDValue Op, SDValue &Mul0, SDValue &Mul1, SDValue &Addend0,
578 isADDADDMUL(SDValue Op, SDValue &Mul0, SDValue &Mul1, SDValue &Addend0,
578 isADDADDMUL(SDValue Op, SDValue &Mul0, SDValue &Mul1, SDValue &Addend0,
578 isADDADDMUL(SDValue Op, SDValue &Mul0, SDValue &Mul1, SDValue &Addend0,
579 SDValue &Addend1, bool requireIntermediatesHaveOneUse)
583 SDValue N0 = Op.getOperand(0);
584 SDValue N1 = Op.getOperand(1);
585 SDValue AddOp;
586 SDValue OtherOp;
631 SDValue XCoreTargetLowering::
634 SDValue Mul;
635 SDValue Other;
646 SDValue LL, RL, AddendL, AddendH;
661 SDValue Hi = DAG.getNode(XCoreISD::MACCU, dl,
664 SDValue Lo(Hi.getNode(), 1);
669 SDValue Hi = DAG.getNode(XCoreISD::MACCS, dl,
672 SDValue Lo(Hi.getNode(), 1);
675 SDValue LH, RH;
680 SDValue Hi = DAG.getNode(XCoreISD::MACCU, dl,
683 SDValue Lo(Hi.getNode(), 1);
691 SDValue XCoreTargetLowering::
699 if (SDValue Result = TryExpandADDWithMul(N, DAG))
705 SDValue LHSL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
708 SDValue LHSH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
711 SDValue RHSL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
714 SDValue RHSH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
721 SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
722 SDValue Lo = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
724 SDValue Carry(Lo.getNode(), 1);
726 SDValue Hi = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
728 SDValue Ignored(Hi.getNode(), 1);
733 SDValue XCoreTargetLowering::
734 LowerVAARG(SDValue Op, SelectionDAG &DAG) const
740 SDValue InChain = Node->getOperand(0);
741 SDValue VAListPtr = Node->getOperand(1);
745 SDValue VAList =
748 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAList,
758 SDValue XCoreTargetLowering::
759 LowerVASTART(SDValue Op, SelectionDAG &DAG) const
766 SDValue Addr = DAG.getFrameIndex(XFI->getVarArgsFrameIndex(), MVT::i32);
771 SDValue XCoreTargetLowering::LowerFRAMEADDR(SDValue Op,
771 SDValue XCoreTargetLowering::LowerFRAMEADDR(SDValue Op,
787 SDValue XCoreTargetLowering::
788 LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
800 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
806 SDValue XCoreTargetLowering::
807 LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const {
815 SDValue XCoreTargetLowering::
816 LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
822 SDValue Chain = Op.getOperand(0);
823 SDValue Offset = Op.getOperand(1);
824 SDValue Handler = Op.getOperand(2);
829 SDValue Stack = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
831 SDValue FrameToArgs = DAG.getNode(XCoreISD::FRAME_TO_ARGS_OFFSET, dl,
841 SDValue OutChains[] = {
854 SDValue XCoreTargetLowering::
855 LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
859 SDValue XCoreTargetLowering::
860 LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const {
861 SDValue Chain = Op.getOperand(0);
862 SDValue Trmp = Op.getOperand(1); // trampoline
863 SDValue FPtr = Op.getOperand(2); // nested function
864 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
879 SDValue OutChains[5];
881 SDValue Addr = Trmp;
913 SDValue XCoreTargetLowering::
914 LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const {
920 SDValue Data =
923 SDValue Crc(Data.getNode(), 1);
924 SDValue Results[] = { Crc, Data };
930 SDValue XCoreTargetLowering::
931 LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const {
936 SDValue XCoreTargetLowering::
937 LowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const {
967 SDValue XCoreTargetLowering::
968 LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const {
1029 SDValue
1031 SmallVectorImpl<SDValue> &InVals) const {
1035 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1037 SDValue Chain = CLI.Chain;
1038 SDValue Callee = CLI.Callee;
1060 static SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1060 static SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1060 static SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1063 SmallVectorImpl<SDValue> &InVals) {
1083 SmallVector<SDValue, 4> MemOpChains;
1088 SDValue Ops[] = { Chain, DAG.getConstant(offset / 4, dl, MVT::i32) };
1089 SDValue load = DAG.getNode(XCoreISD::LDWSP, dl, VTs, Ops);
1106 SDValue XCoreTargetLowering::LowerCCCCallTo(
1107 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
1107 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg,
1109 const SmallVectorImpl<SDValue> &OutVals,
1111 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1137 SmallVector<std::pair<unsigned, SDValue>, 4> RegsToPass;
1138 SmallVector<SDValue, 12> MemOpChains;
1143 SDValue Arg = OutVals[i];
1185 SDValue InFlag;
1205 SmallVector<SDValue, 8> Ops;
1236 struct ArgDataPair { SDValue SDV; ISD::ArgFlagsTy Flags; };
1240 SDValue XCoreTargetLowering::LowerFormalArguments(
1241 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1243 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1259 SDValue XCoreTargetLowering::LowerCCCArguments(
1260 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1262 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1289 SmallVector<SDValue, 4> CFRegNode;
1291 SmallVector<SDValue, 4> MemOps;
1297 SDValue ArgIn;
1334 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1361 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1365 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
1368 SDValue Store =
1396 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1435 SDValue
1436 XCoreTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1439 const SmallVectorImpl<SDValue> &OutVals,
1460 SDValue Flag;
1461 SmallVector<SDValue, 4> RetOps(1, Chain);
1466 SmallVector<SDValue, 4> MemOpChains;
1484 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1589 SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
1600 SDValue OutVal = N->getOperand(3);
1616 SDValue Time = N->getOperand(3);
1634 SDValue N0 = N->getOperand(0);
1635 SDValue N1 = N->getOperand(1);
1636 SDValue N2 = N->getOperand(2);
1647 SDValue Carry = DAG.getConstant(0, dl, VT);
1648 SDValue Result = DAG.getNode(ISD::AND, dl, VT, N2,
1650 SDValue Ops[] = { Result, Carry };
1661 SDValue Carry = DAG.getConstant(0, dl, VT);
1662 SDValue Result = DAG.getNode(ISD::ADD, dl, VT, N0, N2);
1663 SDValue Ops[] = { Result, Carry };
1670 SDValue N0 = N->getOperand(0);
1671 SDValue N1 = N->getOperand(1);
1672 SDValue N2 = N->getOperand(2);
1683 SDValue Borrow = N2;
1684 SDValue Result = DAG.getNode(ISD::SUB, dl, VT,
1686 SDValue Ops[] = { Result, Borrow };
1698 SDValue Borrow = DAG.getConstant(0, dl, VT);
1699 SDValue Result = DAG.getNode(ISD::SUB, dl, VT, N0, N2);
1700 SDValue Ops[] = { Result, Borrow };
1707 SDValue N0 = N->getOperand(0);
1708 SDValue N1 = N->getOperand(1);
1709 SDValue N2 = N->getOperand(2);
1710 SDValue N3 = N->getOperand(3);
1725 SDValue Lo = DAG.getNode(ISD::ADD, dl, VT, N2, N3);
1726 SDValue Ops[] = { Lo, Lo };
1730 SDValue Result =
1732 SDValue Carry(Result.getNode(), 1);
1733 SDValue Ops[] = { Carry, Result };
1743 SDValue Mul0, Mul1, Addend0, Addend1;
1746 SDValue Ignored = DAG.getNode(XCoreISD::LMUL, dl,
1749 SDValue Result(Ignored.getNode(), 1);
1763 SDValue Mul0L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
1765 SDValue Mul1L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
1767 SDValue Addend0L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
1769 SDValue Addend1L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
1771 SDValue Hi = DAG.getNode(XCoreISD::LMUL, dl,
1774 SDValue Lo(Hi.getNode(), 1);
1789 SDValue Chain = ST->getChain();
1815 void XCoreTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
lib/Target/XCore/XCoreISelLowering.h 99 bool isZExtFree(SDValue Val, EVT VT2) const override;
108 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
108 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
113 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
147 SDValue LowerCCCArguments(SDValue Chain, CallingConv::ID CallConv,
147 SDValue LowerCCCArguments(SDValue Chain, CallingConv::ID CallConv,
151 SmallVectorImpl<SDValue> &InVals) const;
152 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
152 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
152 SDValue LowerCCCCallTo(SDValue Chain, SDValue Callee,
156 const SmallVectorImpl<SDValue> &OutVals,
159 SmallVectorImpl<SDValue> &InVals) const;
160 SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
161 SDValue getGlobalAddressWrapper(SDValue GA, const GlobalValue *GV,
161 SDValue getGlobalAddressWrapper(SDValue GA, const GlobalValue *GV,
163 SDValue lowerLoadWordFromAlignedBasePlusOffset(const SDLoc &DL,
164 SDValue Chain, SDValue Base,
164 SDValue Chain, SDValue Base,
169 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
169 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
170 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
170 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
171 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
171 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
172 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
172 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
173 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
173 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
174 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
174 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
175 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
175 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
176 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
176 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
177 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
177 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
178 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
178 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
179 SDValue LowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
179 SDValue LowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
180 SDValue LowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
180 SDValue LowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
181 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
181 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
182 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
182 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
183 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
183 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
184 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
184 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
185 SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
185 SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
186 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
186 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
187 SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
187 SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
188 SDValue LowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
188 SDValue LowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
189 SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
189 SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
199 SDValue TryExpandADDWithMul(SDNode *Op, SelectionDAG &DAG) const;
200 SDValue ExpandADDSUB(SDNode *Op, SelectionDAG &DAG) const;
202 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
204 void computeKnownBitsForTargetNode(const SDValue Op,
210 SDValue
211 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
214 SmallVectorImpl<SDValue> &InVals) const override;
216 SDValue
218 SmallVectorImpl<SDValue> &InVals) const override;
220 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
220 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
222 const SmallVectorImpl<SDValue> &OutVals,
lib/Target/XCore/XCoreSelectionDAGInfo.cpp 18 SDValue XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20 SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
44 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
44 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
lib/Target/XCore/XCoreSelectionDAGInfo.h 24 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
25 SDValue Chain, SDValue Op1, SDValue Op2,
25 SDValue Chain, SDValue Op1, SDValue Op2,
25 SDValue Chain, SDValue Op1, SDValue Op2,
26 SDValue Op3, unsigned Align, bool isVolatile,
unittests/CodeGen/AArch64SelectionDAGTest.cpp 88 auto InVec = DAG->getConstant(0, Loc, InVecVT);
89 auto Op = DAG->getNode(ISD::ZERO_EXTEND_VECTOR_INREG, Loc, OutVecVT, InVec);
102 auto Vec = DAG->getConstant(0, Loc, VecVT);
103 auto ZeroIdx = DAG->getConstant(0, Loc, IdxVT);
104 auto Op = DAG->getNode(ISD::EXTRACT_SUBVECTOR, Loc, VecVT, Vec, ZeroIdx);
118 auto InVec = DAG->getConstant(1, Loc, InVecVT);
119 auto Op = DAG->getNode(ISD::SIGN_EXTEND_VECTOR_INREG, Loc, OutVecVT, InVec);
131 auto Vec = DAG->getConstant(1, Loc, VecVT);
132 auto ZeroIdx = DAG->getConstant(0, Loc, IdxVT);
133 auto Op = DAG->getNode(ISD::EXTRACT_SUBVECTOR, Loc, VecVT, Vec, ZeroIdx);
148 auto Vec = DAG->getConstant(1, Loc, VecVT);
149 auto ZeroIdx = DAG->getConstant(0, Loc, IdxVT);
150 auto Op = DAG->getNode(ISD::EXTRACT_SUBVECTOR, Loc, VecVT, Vec, ZeroIdx);
166 auto UnknownOp = DAG->getRegister(0, IntVT);
167 auto Mask = DAG->getConstant(0x8A, Loc, IntVT);
168 auto N0 = DAG->getNode(ISD::AND, Loc, IntVT, Mask, UnknownOp);
169 auto N1 = DAG->getConstant(0x55, Loc, IntVT);
170 auto Op = DAG->getNode(ISD::ADD, Loc, IntVT, N0, N1);
187 auto N0 = DAG->getConstant(0x55, Loc, IntVT);
188 auto UnknownOp = DAG->getRegister(0, IntVT);
189 auto Mask = DAG->getConstant(0x2e, Loc, IntVT);
190 auto N1 = DAG->getNode(ISD::AND, Loc, IntVT, Mask, UnknownOp);
191 auto Op = DAG->getNode(ISD::SUB, Loc, IntVT, N0, N1);
unittests/CodeGen/TypeTraitsTest.cpp 21 static_assert(is_trivially_copyable<SDValue>::value, "trivially copyable");
usr/include/c++/7.4.0/bits/alloc_traits.h 387 using allocator_type = allocator<_Tp>;
389 using value_type = _Tp;
392 using pointer = _Tp*;
395 using const_pointer = const _Tp*;
474 construct(allocator_type& __a, _Up* __p, _Args&&... __args)
474 construct(allocator_type& __a, _Up* __p, _Args&&... __args)
475 { __a.construct(__p, std::forward<_Args>(__args)...); }
486 destroy(allocator_type& __a, _Up* __p)
usr/include/c++/7.4.0/bits/allocator.h 108 class allocator: public __allocator_base<_Tp>
113 typedef _Tp* pointer;
114 typedef const _Tp* const_pointer;
115 typedef _Tp& reference;
116 typedef const _Tp& const_reference;
117 typedef _Tp value_type;
137 allocator(const allocator<_Tp1>&) throw() { }
usr/include/c++/7.4.0/bits/move.h 46 inline _GLIBCXX_CONSTEXPR _Tp*
47 __addressof(_Tp& __r) _GLIBCXX_NOEXCEPT
72 constexpr _Tp&&
73 forward(typename std::remove_reference<_Tp>::type& __t) noexcept
83 constexpr _Tp&&
84 forward(typename std::remove_reference<_Tp>::type&& __t) noexcept
98 move(_Tp&& __t) noexcept
104 : public __and_<__not_<is_nothrow_move_constructible<_Tp>>,
105 is_copy_constructible<_Tp>>::type { };
136 inline _GLIBCXX17_CONSTEXPR _Tp*
137 addressof(_Tp& __r) noexcept
143 const _Tp* addressof(const _Tp&&) = delete;
143 const _Tp* addressof(const _Tp&&) = delete;
184 typename enable_if<__and_<__not_<__is_tuple_like<_Tp>>,
185 is_move_constructible<_Tp>,
186 is_move_assignable<_Tp>>::value>::type
187 swap(_Tp& __a, _Tp& __b)
187 swap(_Tp& __a, _Tp& __b)
198 _Tp __tmp = _GLIBCXX_MOVE(__a);
usr/include/c++/7.4.0/bits/ptr_traits.h 126 typedef _Tp* pointer;
128 typedef _Tp element_type;
141 pointer_to(__make_not_void<element_type>& __r) noexcept
141 pointer_to(__make_not_void<element_type>& __r) noexcept
usr/include/c++/7.4.0/bits/std_function.h 62 : std::unary_function<_T1, _Res> { };
299 _M_invoke(const _Any_data& __functor, _ArgTypes&&... __args)
302 std::forward<_ArgTypes>(__args)...);
314 _M_invoke(const _Any_data& __functor, _ArgTypes&&... __args)
317 std::forward<_ArgTypes>(__args)...);
390 : public _Maybe_unary_or_binary_function<_Res, _ArgTypes...>,
595 _Res operator()(_ArgTypes... __args) const;
628 using _Invoker_type = _Res (*)(const _Any_data&, _ArgTypes&&...);
689 typedef _Function_handler<_Res(_ArgTypes...), _Functor> _My_handler;
706 return _M_invoker(_M_functor, std::forward<_ArgTypes>(__args)...);
usr/include/c++/7.4.0/bits/stl_algo.h 3900 const _Tp& __val)
4397 _Predicate __pred, const _Tp& __new_value)
usr/include/c++/7.4.0/bits/stl_construct.h 74 _Construct(_T1* __p, _Args&&... __args)
74 _Construct(_T1* __p, _Args&&... __args)
75 { ::new(static_cast<void*>(__p)) _T1(std::forward<_Args>(__args)...); }
75 { ::new(static_cast<void*>(__p)) _T1(std::forward<_Args>(__args)...); }
204 allocator<_Tp>&)
usr/include/c++/7.4.0/bits/stl_function.h 108 typedef _Arg argument_type;
111 typedef _Result result_type;
121 typedef _Arg1 first_argument_type;
124 typedef _Arg2 second_argument_type;
381 struct less : public binary_function<_Tp, _Tp, bool>
381 struct less : public binary_function<_Tp, _Tp, bool>
385 operator()(const _Tp& __x, const _Tp& __y) const
385 operator()(const _Tp& __x, const _Tp& __y) const
870 : public unary_function<_Tp,_Tp>
870 : public unary_function<_Tp,_Tp>
872 _Tp&
873 operator()(_Tp& __x) const
876 const _Tp&
877 operator()(const _Tp& __x) const
usr/include/c++/7.4.0/bits/stl_iterator.h 1224 __make_move_if_noexcept_iterator(_Tp* __i)
usr/include/c++/7.4.0/bits/stl_iterator_base_types.h 123 typedef _Tp value_type;
181 typedef _Tp value_type;
183 typedef _Tp* pointer;
184 typedef _Tp& reference;
192 typedef _Tp value_type;
194 typedef const _Tp* pointer;
195 typedef const _Tp& reference;
usr/include/c++/7.4.0/bits/stl_list.h 570 _M_create_node(_Args&&... __args)
1081 emplace_front(_Args&&... __args)
1131 emplace_back(_Args&&... __args)
1799 _M_insert(iterator __position, _Args&&... __args)
usr/include/c++/7.4.0/bits/stl_pair.h 100 return __and_<is_constructible<_T1, const _U1&>,
100 return __and_<is_constructible<_T1, const _U1&>,
101 is_constructible<_T2, const _U2&>>::value;
101 is_constructible<_T2, const _U2&>>::value;
107 return __and_<is_convertible<const _U1&, _T1>,
107 return __and_<is_convertible<const _U1&, _T1>,
108 is_convertible<const _U2&, _T2>>::value;
108 is_convertible<const _U2&, _T2>>::value;
114 return __and_<is_constructible<_T1, _U1&&>,
114 return __and_<is_constructible<_T1, _U1&&>,
115 is_constructible<_T2, _U2&&>>::value;
115 is_constructible<_T2, _U2&&>>::value;
121 return __and_<is_convertible<_U1&&, _T1>,
121 return __and_<is_convertible<_U1&&, _T1>,
122 is_convertible<_U2&&, _T2>>::value;
122 is_convertible<_U2&&, _T2>>::value;
128 using __do_converts = __and_<is_convertible<const _U1&, _T1>,
128 using __do_converts = __and_<is_convertible<const _U1&, _T1>,
129 is_convertible<_U2&&, _T2>>;
129 is_convertible<_U2&&, _T2>>;
133 return __and_<is_constructible<_T1, const _U1&>,
133 return __and_<is_constructible<_T1, const _U1&>,
134 is_constructible<_T2, _U2&&>,
134 is_constructible<_T2, _U2&&>,
142 using __do_converts = __and_<is_convertible<_U1&&, _T1>,
142 using __do_converts = __and_<is_convertible<_U1&&, _T1>,
143 is_convertible<const _U2&, _T2>>;
143 is_convertible<const _U2&, _T2>>;
147 return __and_<is_constructible<_T1, _U1&&>,
147 return __and_<is_constructible<_T1, _U1&&>,
148 is_constructible<_T2, const _U2&&>,
148 is_constructible<_T2, const _U2&&>,
209 : private __pair_base<_T1, _T2>
209 : private __pair_base<_T1, _T2>
211 typedef _T1 first_type; /// @c first_type is the first bound type
212 typedef _T2 second_type; /// @c second_type is the second bound type
214 _T1 first; /// @c first is a copy of the first object
215 _T2 second; /// @c second is a copy of the second object
252 using _PCCP = _PCC<true, _T1, _T2>;
252 using _PCCP = _PCC<true, _T1, _T2>;
260 constexpr pair(const _T1& __a, const _T2& __b)
260 constexpr pair(const _T1& __a, const _T2& __b)
269 explicit constexpr pair(const _T1& __a, const _T2& __b)
269 explicit constexpr pair(const _T1& __a, const _T2& __b)
283 _T1, _T2>;
283 _T1, _T2>;
291 constexpr pair(const pair<_U1, _U2>& __p)
291 constexpr pair(const pair<_U1, _U2>& __p)
311 constexpr pair(_U1&& __x, const _T2& __y)
311 constexpr pair(_U1&& __x, const _T2& __y)
318 explicit constexpr pair(_U1&& __x, const _T2& __y)
325 constexpr pair(const _T1& __x, _U2&& __y)
325 constexpr pair(const _T1& __x, _U2&& __y)
332 explicit pair(const _T1& __x, _U2&& __y)
341 constexpr pair(_U1&& __x, _U2&& __y)
341 constexpr pair(_U1&& __x, _U2&& __y)
342 : first(std::forward<_U1>(__x)), second(std::forward<_U2>(__y)) { }
342 : first(std::forward<_U1>(__x)), second(std::forward<_U2>(__y)) { }
360 constexpr pair(pair<_U1, _U2>&& __p)
360 constexpr pair(pair<_U1, _U2>&& __p)
361 : first(std::forward<_U1>(__p.first)),
362 second(std::forward<_U2>(__p.second)) { }
379 __and_<is_copy_assignable<_T1>,
380 is_copy_assignable<_T2>>::value,
390 __and_<is_move_assignable<_T1>,
391 is_move_assignable<_T2>>::value,
402 typename enable_if<__and_<is_assignable<_T1&, const _U1&>,
402 typename enable_if<__and_<is_assignable<_T1&, const _U1&>,
403 is_assignable<_T2&, const _U2&>>::value,
403 is_assignable<_T2&, const _U2&>>::value,
405 operator=(const pair<_U1, _U2>& __p)
405 operator=(const pair<_U1, _U2>& __p)
413 typename enable_if<__and_<is_assignable<_T1&, _U1&&>,
413 typename enable_if<__and_<is_assignable<_T1&, _U1&&>,
414 is_assignable<_T2&, _U2&&>>::value,
414 is_assignable<_T2&, _U2&&>>::value,
416 operator=(pair<_U1, _U2>&& __p)
416 operator=(pair<_U1, _U2>&& __p)
448 operator==(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
448 operator==(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
448 operator==(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
448 operator==(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
495 swap(pair<_T1, _T2>& __x, pair<_T1, _T2>& __y)
522 constexpr pair<typename __decay_and_strip<_T1>::__type,
523 typename __decay_and_strip<_T2>::__type>
524 make_pair(_T1&& __x, _T2&& __y)
524 make_pair(_T1&& __x, _T2&& __y)
526 typedef typename __decay_and_strip<_T1>::__type __ds_type1;
527 typedef typename __decay_and_strip<_T2>::__type __ds_type2;
529 return __pair_type(std::forward<_T1>(__x), std::forward<_T2>(__y));
529 return __pair_type(std::forward<_T1>(__x), std::forward<_T2>(__y));
usr/include/c++/7.4.0/bits/stl_set.h 110 typedef _Key key_type;
111 typedef _Key value_type;
119 rebind<_Key>::other _Key_alloc_type;
usr/include/c++/7.4.0/bits/stl_tree.h 218 typedef _Rb_tree_node<_Val>* _Link_type;
231 __gnu_cxx::__aligned_membuf<_Val> _M_storage;
233 _Val*
237 const _Val*
258 typedef _Tp value_type;
259 typedef _Tp& reference;
260 typedef _Tp* pointer;
265 typedef _Rb_tree_iterator<_Tp> _Self;
267 typedef _Rb_tree_node<_Tp>* _Link_type;
328 typedef _Tp value_type;
329 typedef const _Tp& reference;
330 typedef const _Tp* pointer;
332 typedef _Rb_tree_iterator<_Tp> iterator;
337 typedef _Rb_tree_const_iterator<_Tp> _Self;
339 typedef const _Rb_tree_node<_Tp>* _Link_type;
447 rebind<_Rb_tree_node<_Val> >::other _Node_allocator;
454 typedef _Rb_tree_node<_Val>* _Link_type;
455 typedef const _Rb_tree_node<_Val>* _Const_Link_type;
554 operator()(_Arg&& __arg) const
563 typedef _Key key_type;
564 typedef _Val value_type;
621 _M_construct_node(_Link_type __node, _Args&&... __args)
625 ::new(__node) _Rb_tree_node<_Val>;
640 _M_create_node(_Args&&... __args)
758 static const _Key&
782 static const _Key&
834 _M_insert_(_Base_ptr __x, _Base_ptr __y, _Arg&& __v, _NodeGen&);
894 const _Key& __k);
898 const _Key& __k) const;
902 const _Key& __k);
906 const _Key& __k) const;
1011 _M_insert_unique(_Arg&& __x);
usr/include/c++/7.4.0/bits/stl_uninitialized.h 144 const _Tp& __x)
182 const _Tp& __x)
204 const _Tp& __x)
244 uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x)
288 _ForwardIterator __result, allocator<_Tp>&)
644 allocator<_Tp>&)
usr/include/c++/7.4.0/bits/stl_vector.h 77 rebind<_Tp>::other _Tp_alloc_type;
216 class vector : protected _Vector_base<_Tp, _Alloc>
227 typedef _Vector_base<_Tp, _Alloc> _Base;
232 typedef _Tp value_type;
919 _Tp*
923 const _Tp*
962 emplace_back(_Args&&... __args);
1455 _Temporary_value(vector* __vec, _Args&&... __args) : _M_this(__vec)
1472 typename aligned_storage<sizeof(_Tp), alignof(_Tp)>::type __buf;
1472 typename aligned_storage<sizeof(_Tp), alignof(_Tp)>::type __buf;
1479 _M_insert_aux(iterator __position, _Arg&& __arg);
1483 _M_realloc_insert(iterator __position, _Args&&... __args);
1561 _Up*
1562 _M_data_ptr(_Up* __ptr) const _GLIBCXX_NOEXCEPT
1649 swap(vector<_Tp, _Alloc>& __x, vector<_Tp, _Alloc>& __y)
1649 swap(vector<_Tp, _Alloc>& __x, vector<_Tp, _Alloc>& __y)
usr/include/c++/7.4.0/bits/vector.tcc 101 std::forward<_Args>(__args)...);
105 _M_realloc_insert(end(), std::forward<_Args>(__args)...);
386 *__position = std::forward<_Arg>(__arg);
418 std::forward<_Args>(__args)...);
usr/include/c++/7.4.0/ext/aligned_buffer.h 52 struct _Tp2 { _Tp _M_t; };
54 alignas(__alignof__(_Tp2::_M_t)) unsigned char _M_storage[sizeof(_Tp)];
69 _Tp*
73 const _Tp*
usr/include/c++/7.4.0/ext/alloc_traits.h 117 { typedef typename _Base_type::template rebind_alloc<_Tp> other; };
usr/include/c++/7.4.0/ext/new_allocator.h 63 typedef _Tp* pointer;
64 typedef const _Tp* const_pointer;
65 typedef _Tp& reference;
66 typedef const _Tp& const_reference;
67 typedef _Tp value_type;
84 new_allocator(const new_allocator<_Tp1>&) _GLIBCXX_USE_NOEXCEPT { }
111 return static_cast<_Tp*>(::operator new(__n * sizeof(_Tp)));
130 { return size_t(-1) / sizeof(_Tp); }
135 construct(_Up* __p, _Args&&... __args)
135 construct(_Up* __p, _Args&&... __args)
136 { ::new((void *)__p) _Up(std::forward<_Args>(__args)...); }
136 { ::new((void *)__p) _Up(std::forward<_Args>(__args)...); }
140 destroy(_Up* __p) { __p->~_Up(); }
usr/include/c++/7.4.0/initializer_list 50 typedef _E value_type;
51 typedef const _E& reference;
52 typedef const _E& const_reference;
54 typedef const _E* iterator;
55 typedef const _E* const_iterator;
usr/include/c++/7.4.0/tuple 56 struct __is_empty_non_tuple : is_empty<_Tp> { };
125 constexpr _Head_base(const _Head& __h)
132 constexpr _Head_base(_UHead&& __h)
133 : _M_head_impl(std::forward<_UHead>(__h)) { }
159 static constexpr _Head&
162 static constexpr const _Head&
165 _Head _M_head_impl;
186 : public _Tuple_impl<_Idx + 1, _Tail...>,
187 private _Head_base<_Idx, _Head>
191 typedef _Tuple_impl<_Idx + 1, _Tail...> _Inherited;
192 typedef _Head_base<_Idx, _Head> _Base;
194 static constexpr _Head&
197 static constexpr const _Head&
210 constexpr _Tuple_impl(const _Head& __head, const _Tail&... __tail)
210 constexpr _Tuple_impl(const _Head& __head, const _Tail&... __tail)
216 constexpr _Tuple_impl(_UHead&& __head, _UTail&&... __tail)
216 constexpr _Tuple_impl(_UHead&& __head, _UTail&&... __tail)
227 _Base(std::forward<_Head>(_M_head(__in))) { }
230 constexpr _Tuple_impl(const _Tuple_impl<_Idx, _UElements...>& __in)
235 constexpr _Tuple_impl(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
235 constexpr _Tuple_impl(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
242 _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a)
248 const _Head& __head, const _Tail&... __tail)
248 const _Head& __head, const _Tail&... __tail)
262 _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
268 _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
350 static constexpr _Head&
353 static constexpr const _Head&
360 constexpr _Tuple_impl(const _Head& __head)
365 constexpr _Tuple_impl(_UHead&& __head)
390 const _Head& __head)
473 return __and_<is_constructible<_Elements, const _UElements&>...>::value;
473 return __and_<is_constructible<_Elements, const _UElements&>...>::value;
479 return __and_<is_convertible<const _UElements&, _Elements>...>::value;
479 return __and_<is_convertible<const _UElements&, _Elements>...>::value;
485 return __and_<is_constructible<_Elements, _UElements&&>...>::value;
485 return __and_<is_constructible<_Elements, _UElements&&>...>::value;
491 return __and_<is_convertible<_UElements&&, _Elements>...>::value;
491 return __and_<is_convertible<_UElements&&, _Elements>...>::value;
508 return __not_<is_same<tuple<_Elements...>,
556 class tuple : public _Tuple_impl<0, _Elements...>
558 typedef _Tuple_impl<0, _Elements...> _Inherited;
598 _Elements...>;
608 constexpr tuple(const _Elements&... __elements)
619 explicit constexpr tuple(const _Elements&... __elements)
628 _Elements...>;
636 _Elements...>;
646 constexpr tuple(_UElements&&... __elements)
668 _Elements...>;
730 const _Elements&... __elements)
741 const _Elements&... __elements)
767 tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in)
771 tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in)
947 constexpr tuple(const _T1& __a1, const _T2& __a2)
947 constexpr tuple(const _T1& __a1, const _T2& __a2)
956 explicit constexpr tuple(const _T1& __a1, const _T2& __a2)
956 explicit constexpr tuple(const _T1& __a1, const _T2& __a2)
971 constexpr tuple(_U1&& __a1, _U2&& __a2)
971 constexpr tuple(_U1&& __a1, _U2&& __a2)
1066 tuple(allocator_arg_t __tag, const _Alloc& __a)
1078 const _T1& __a1, const _T2& __a2)
1078 const _T1& __a1, const _T2& __a2)
1090 const _T1& __a1, const _T2& __a2)
1090 const _T1& __a1, const _T2& __a2)
1241 operator=(const pair<_U1, _U2>& __in)
1241 operator=(const pair<_U1, _U2>& __in)
1250 operator=(pair<_U1, _U2>&& __in)
1250 operator=(pair<_U1, _U2>&& __in)
1252 this->_M_head(*this) = std::forward<_U1>(__in.first);
1253 this->_M_tail(*this)._M_head(*this) = std::forward<_U2>(__in.second);
1280 : tuple_element<__i - 1, tuple<_Tail...> > { };
1288 typedef _Head type;
1302 constexpr _Head&
1303 __get_helper(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
1303 __get_helper(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
1304 { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }
1304 { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }
1307 constexpr const _Head&
1308 __get_helper(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
1308 __get_helper(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
1313 constexpr __tuple_element_t<__i, tuple<_Elements...>>&
1313 constexpr __tuple_element_t<__i, tuple<_Elements...>>&
1313 constexpr __tuple_element_t<__i, tuple<_Elements...>>&
1314 get(tuple<_Elements...>& __t) noexcept
1319 constexpr const __tuple_element_t<__i, tuple<_Elements...>>&
1319 constexpr const __tuple_element_t<__i, tuple<_Elements...>>&
1319 constexpr const __tuple_element_t<__i, tuple<_Elements...>>&
1320 get(const tuple<_Elements...>& __t) noexcept
1325 constexpr __tuple_element_t<__i, tuple<_Elements...>>&&
1325 constexpr __tuple_element_t<__i, tuple<_Elements...>>&&
1325 constexpr __tuple_element_t<__i, tuple<_Elements...>>&&
1326 get(tuple<_Elements...>&& __t) noexcept
1448 make_tuple(_Elements&&... __args)
1588 constexpr tuple<_Elements&...>
1589 tie(_Elements&... __args) noexcept
usr/include/c++/7.4.0/type_traits 215 : public __is_void_helper<typename remove_cv<_Tp>::type>::type
326 : public __is_integral_helper<typename remove_cv<_Tp>::type>::type
354 : public __is_floating_point_helper<typename remove_cv<_Tp>::type>::type
381 : public __is_pointer_helper<typename remove_cv<_Tp>::type>::type
567 : public __is_null_pointer_helper<typename remove_cv<_Tp>::type>::type
581 : public __or_<is_lvalue_reference<_Tp>,
582 is_rvalue_reference<_Tp>>::type
588 : public __or_<is_integral<_Tp>, is_floating_point<_Tp>>::type
588 : public __or_<is_integral<_Tp>, is_floating_point<_Tp>>::type
601 : public __not_<__or_<is_function<_Tp>, is_reference<_Tp>,
601 : public __not_<__or_<is_function<_Tp>, is_reference<_Tp>,
602 is_void<_Tp>>>::type
611 : public __or_<is_arithmetic<_Tp>, is_enum<_Tp>, is_pointer<_Tp>,
611 : public __or_<is_arithmetic<_Tp>, is_enum<_Tp>, is_pointer<_Tp>,
611 : public __or_<is_arithmetic<_Tp>, is_enum<_Tp>, is_pointer<_Tp>,
612 is_member_pointer<_Tp>, is_null_pointer<_Tp>>::type
612 is_member_pointer<_Tp>, is_null_pointer<_Tp>>::type
631 : public __is_member_pointer_helper<typename remove_cv<_Tp>::type>::type
638 : public __or_<is_object<_Tp>, is_reference<_Tp>>::type
638 : public __or_<is_object<_Tp>, is_reference<_Tp>>::type
762 typename add_rvalue_reference<_Tp>::type declval() noexcept;
777 : public __and_<is_array<_Tp>, __not_<extent<_Tp>>>
777 : public __and_<is_array<_Tp>, __not_<extent<_Tp>>>
798 typedef decltype(__test<_Tp>(0)) type;
811 remove_all_extents<_Tp>::type>::type
825 : public __is_destructible_safe<_Tp>::type
889 typedef decltype(__test<_Tp>(0)) type;
894 : public __and_<__not_<is_void<_Tp>>,
895 __is_default_constructible_impl<_Tp>>
915 : public __is_default_constructible_atom<_Tp>::type
921 : public __is_default_constructible_safe<_Tp>::type
984 typedef decltype(__test<_Tp, _Arg>(0)) type;
989 : public __and_<is_destructible<_Tp>,
990 __is_direct_constructible_impl<_Tp, _Arg>>
1072 __is_direct_constructible_ref_cast<_Tp, _Arg>,
1073 __is_direct_constructible_new_safe<_Tp, _Arg>
1079 : public __is_direct_constructible_new<_Tp, _Arg>::type
1119 : public __is_direct_constructible<_Tp, _Arg>
1130 : public __is_constructible_impl<_Tp, _Args...>::type
1142 : public is_constructible<_Tp, const _Tp&>
1142 : public is_constructible<_Tp, const _Tp&>
1148 : public __is_copy_constructible_impl<_Tp>
1160 : public is_constructible<_Tp, _Tp&&>
1160 : public is_constructible<_Tp, _Tp&&>
1166 : public __is_move_constructible_impl<_Tp>
1215 : public __and_<is_constructible<_Tp, _Args...>,
1216 __is_nt_constructible_impl<_Tp, _Args...>>
1246 : public is_nothrow_constructible<_Tp, _Tp&&>
1246 : public is_nothrow_constructible<_Tp, _Tp&&>
1252 : public __is_nothrow_move_constructible_impl<_Tp>
1286 : public is_assignable<_Tp&, const _Tp&>
1286 : public is_assignable<_Tp&, const _Tp&>
1292 : public __is_copy_assignable_impl<_Tp>
1304 : public is_assignable<_Tp&, _Tp&&>
1304 : public is_assignable<_Tp&, _Tp&&>
1310 : public __is_move_assignable_impl<_Tp>
1352 : public is_nothrow_assignable<_Tp&, _Tp&&>
1352 : public is_nothrow_assignable<_Tp&, _Tp&&>
1358 : public __is_nt_move_assignable_impl<_Tp>
1377 static void __helper(const _Tp&);
1380 static true_type __test(const _Tp&,
1381 decltype(__helper<const _Tp&>({}))* = 0);
1390 typedef decltype(__test(declval<_Tp>())) type;
1395 : public __is_implicitly_default_constructible_impl<_Tp>::type
1400 : public __and_<is_default_constructible<_Tp>,
1401 __is_implicitly_default_constructible_safe<_Tp>>
1526 static void __test_aux(_To1);
1538 typedef decltype(__test<_From, _To>(0)) type;
1545 : public __is_convertible_helper<_From, _To>::type
1554 { typedef _Tp type; };
1558 { typedef _Tp type; };
1563 { typedef _Tp type; };
1574 remove_const<typename remove_volatile<_Tp>::type>::type type;
1629 { typedef _Tp type; };
1633 { typedef _Tp type; };
1659 { typedef _Tp&& type; };
1664 : public __add_rvalue_reference_helper<_Tp>
1955 { typedef _Tp type; };
2104 { typedef typename remove_cv<_Up>::type __type; };
2118 typedef typename remove_reference<_Tp>::type __remove_type;
2131 typedef _Tp __type;
2144 typename decay<_Tp>::type>::__type __type;
2171 { typedef _Iffalse type; };
2253 inline typename add_rvalue_reference<_Tp>::type
2428 std::declval<_Fn>()(std::declval<_Args>()...)
2439 typedef decltype(_S_test<_Functor, _ArgTypes...>(0)) type;
2452 _Functor, _ArgTypes...
2458 : public __invoke_result<_Functor, _ArgTypes...>
2574 typename remove_reference<_Tp>::type>::type>::type
2609 noexcept(swap(std::declval<_Tp&>(), std::declval<_Tp&>()))
2609 noexcept(swap(std::declval<_Tp&>(), std::declval<_Tp&>()))
2629 typedef decltype(__test<_Tp>(0)) type;
2639 : public __is_nothrow_swappable_impl<_Tp>::type