1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
| //===-------- X86PadShortFunction.cpp - pad short functions -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the pass which will pad short functions to prevent
// a stall if a function returns before the return address is ready. This
// is needed for some Intel Atom processors.
//
//===----------------------------------------------------------------------===//
#include "X86.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/IR/Function.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
#define DEBUG_TYPE "x86-pad-short-functions"
STATISTIC(NumBBsPadded, "Number of basic blocks padded");
namespace {
struct VisitedBBInfo {
// HasReturn - Whether the BB contains a return instruction
bool HasReturn;
// Cycles - Number of cycles until return if HasReturn is true, otherwise
// number of cycles until end of the BB
unsigned int Cycles;
VisitedBBInfo() : HasReturn(false), Cycles(0) {}
VisitedBBInfo(bool HasReturn, unsigned int Cycles)
: HasReturn(HasReturn), Cycles(Cycles) {}
};
struct PadShortFunc : public MachineFunctionPass {
static char ID;
PadShortFunc() : MachineFunctionPass(ID)
, Threshold(4) {}
bool runOnMachineFunction(MachineFunction &MF) override;
MachineFunctionProperties getRequiredProperties() const override {
return MachineFunctionProperties().set(
MachineFunctionProperties::Property::NoVRegs);
}
StringRef getPassName() const override {
return "X86 Atom pad short functions";
}
private:
void findReturns(MachineBasicBlock *MBB,
unsigned int Cycles = 0);
bool cyclesUntilReturn(MachineBasicBlock *MBB,
unsigned int &Cycles);
void addPadding(MachineBasicBlock *MBB,
MachineBasicBlock::iterator &MBBI,
unsigned int NOOPsToAdd);
const unsigned int Threshold;
// ReturnBBs - Maps basic blocks that return to the minimum number of
// cycles until the return, starting from the entry block.
DenseMap<MachineBasicBlock*, unsigned int> ReturnBBs;
// VisitedBBs - Cache of previously visited BBs.
DenseMap<MachineBasicBlock*, VisitedBBInfo> VisitedBBs;
TargetSchedModel TSM;
};
char PadShortFunc::ID = 0;
}
FunctionPass *llvm::createX86PadShortFunctions() {
return new PadShortFunc();
}
/// runOnMachineFunction - Loop over all of the basic blocks, inserting
/// NOOP instructions before early exits.
bool PadShortFunc::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
if (MF.getFunction().hasOptSize())
return false;
if (!MF.getSubtarget<X86Subtarget>().padShortFunctions())
return false;
TSM.init(&MF.getSubtarget());
// Search through basic blocks and mark the ones that have early returns
ReturnBBs.clear();
VisitedBBs.clear();
findReturns(&MF.front());
bool MadeChange = false;
// Pad the identified basic blocks with NOOPs
for (DenseMap<MachineBasicBlock*, unsigned int>::iterator I = ReturnBBs.begin();
I != ReturnBBs.end(); ++I) {
MachineBasicBlock *MBB = I->first;
unsigned Cycles = I->second;
if (Cycles < Threshold) {
// BB ends in a return. Skip over any DBG_VALUE instructions
// trailing the terminator.
assert(MBB->size() > 0 &&
"Basic block should contain at least a RET but is empty");
MachineBasicBlock::iterator ReturnLoc = --MBB->end();
while (ReturnLoc->isDebugInstr())
--ReturnLoc;
assert(ReturnLoc->isReturn() && !ReturnLoc->isCall() &&
"Basic block does not end with RET");
addPadding(MBB, ReturnLoc, Threshold - Cycles);
NumBBsPadded++;
MadeChange = true;
}
}
return MadeChange;
}
/// findReturn - Starting at MBB, follow control flow and add all
/// basic blocks that contain a return to ReturnBBs.
void PadShortFunc::findReturns(MachineBasicBlock *MBB, unsigned int Cycles) {
// If this BB has a return, note how many cycles it takes to get there.
bool hasReturn = cyclesUntilReturn(MBB, Cycles);
if (Cycles >= Threshold)
return;
if (hasReturn) {
ReturnBBs[MBB] = std::max(ReturnBBs[MBB], Cycles);
return;
}
// Follow branches in BB and look for returns
for (MachineBasicBlock::succ_iterator I = MBB->succ_begin();
I != MBB->succ_end(); ++I) {
if (*I == MBB)
continue;
findReturns(*I, Cycles);
}
}
/// cyclesUntilReturn - return true if the MBB has a return instruction,
/// and return false otherwise.
/// Cycles will be incremented by the number of cycles taken to reach the
/// return or the end of the BB, whichever occurs first.
bool PadShortFunc::cyclesUntilReturn(MachineBasicBlock *MBB,
unsigned int &Cycles) {
// Return cached result if BB was previously visited
DenseMap<MachineBasicBlock*, VisitedBBInfo>::iterator it
= VisitedBBs.find(MBB);
if (it != VisitedBBs.end()) {
VisitedBBInfo BBInfo = it->second;
Cycles += BBInfo.Cycles;
return BBInfo.HasReturn;
}
unsigned int CyclesToEnd = 0;
for (MachineInstr &MI : *MBB) {
// Mark basic blocks with a return instruction. Calls to other
// functions do not count because the called function will be padded,
// if necessary.
if (MI.isReturn() && !MI.isCall()) {
VisitedBBs[MBB] = VisitedBBInfo(true, CyclesToEnd);
Cycles += CyclesToEnd;
return true;
}
CyclesToEnd += TSM.computeInstrLatency(&MI);
}
VisitedBBs[MBB] = VisitedBBInfo(false, CyclesToEnd);
Cycles += CyclesToEnd;
return false;
}
/// addPadding - Add the given number of NOOP instructions to the function
/// just prior to the return at MBBI
void PadShortFunc::addPadding(MachineBasicBlock *MBB,
MachineBasicBlock::iterator &MBBI,
unsigned int NOOPsToAdd) {
DebugLoc DL = MBBI->getDebugLoc();
unsigned IssueWidth = TSM.getIssueWidth();
for (unsigned i = 0, e = IssueWidth * NOOPsToAdd; i != e; ++i)
BuildMI(*MBB, MBBI, DL, TSM.getInstrInfo()->get(X86::NOOP));
}
|