1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
| //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
#include "Compiler.h"
#include "Diagnostics.h"
#include "Function.h"
#include "GlobalCompilationDatabase.h"
#include "Path.h"
#include "Threading.h"
#include "index/CanonicalIncludes.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
namespace clang {
namespace clangd {
class ParsedAST;
struct PreambleData;
/// Returns a number of a default async threads to use for TUScheduler.
/// Returned value is always >= 1 (i.e. will not cause requests to be processed
/// synchronously).
unsigned getDefaultAsyncThreadsCount();
struct InputsAndAST {
const ParseInputs &Inputs;
ParsedAST *
};
struct InputsAndPreamble {
llvm::StringRef Contents;
const tooling::CompileCommand &Command;
// This can be nullptr if no preamble is availble.
const PreambleData *Preamble;
};
/// Determines whether diagnostics should be generated for a file snapshot.
enum class WantDiagnostics {
Yes, /// Diagnostics must be generated for this snapshot.
No, /// Diagnostics must not be generated for this snapshot.
Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
/// within a bounded amount of time.
};
/// Configuration of the AST retention policy. This only covers retention of
/// *idle* ASTs. If queue has operations requiring the AST, they might be
/// kept in memory.
struct ASTRetentionPolicy {
/// Maximum number of ASTs to be retained in memory when there are no pending
/// requests for them.
unsigned MaxRetainedASTs = 3;
};
struct TUAction {
enum State {
Queued, // The TU is pending in the thread task queue to be built.
RunningAction, // Starting running actions on the TU.
BuildingPreamble, // The preamble of the TU is being built.
BuildingFile, // The TU is being built. It is only emitted when building
// the AST for diagnostics in write action (update).
Idle, // Indicates the worker thread is idle, and ready to run any upcoming
// actions.
};
TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
State S;
/// The name of the action currently running, e.g. Update, GoToDef, Hover.
/// Empty if we are in the idle state.
std::string Name;
};
// Internal status of the TU in TUScheduler.
struct TUStatus {
struct BuildDetails {
/// Indicates whether clang failed to build the TU.
bool BuildFailed = false;
/// Indicates whether we reused the prebuilt AST.
bool ReuseAST = false;
};
/// Serialize this to an LSP file status item.
FileStatus render(PathRef File) const;
TUAction Action;
BuildDetails Details;
};
class ParsingCallbacks {
public:
virtual ~ParsingCallbacks() = default;
/// Called on the AST that was built for emitting the preamble. The built AST
/// contains only AST nodes from the #include directives at the start of the
/// file. AST node in the current file should be observed on onMainAST call.
virtual void onPreambleAST(PathRef Path, ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
const CanonicalIncludes &) {}
/// The argument function is run under the critical section guarding against
/// races when closing the files.
using PublishFn = llvm::function_ref<void(llvm::function_ref<void()>)>;
/// Called on the AST built for the file itself. Note that preamble AST nodes
/// are not deserialized and should be processed in the onPreambleAST call
/// instead.
/// The \p AST always contains all AST nodes for the main file itself, and
/// only a portion of the AST nodes deserialized from the preamble. Note that
/// some nodes from the preamble may have been deserialized and may also be
/// accessed from the main file AST, e.g. redecls of functions from preamble,
/// etc. Clients are expected to process only the AST nodes from the main file
/// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
/// optimal performance.
///
/// When information about the file (diagnostics, syntax highlighting) is
/// published to clients, this should be wrapped in Publish, e.g.
/// void onMainAST(...) {
/// Highlights = computeHighlights();
/// Publish([&] { notifyHighlights(Path, Highlights); });
/// }
/// This guarantees that clients will see results in the correct sequence if
/// the file is concurrently closed and/or reopened. (The lambda passed to
/// Publish() may never run in this case).
virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) {}
/// Called whenever the AST fails to build. \p Diags will have the diagnostics
/// that led to failure.
virtual void onFailedAST(PathRef Path, std::vector<Diag> Diags,
PublishFn Publish) {}
/// Called whenever the TU status is updated.
virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
};
/// Handles running tasks for ClangdServer and managing the resources (e.g.,
/// preambles and ASTs) for opened files.
/// TUScheduler is not thread-safe, only one thread should be providing updates
/// and scheduling tasks.
/// Callbacks are run on a threadpool and it's appropriate to do slow work in
/// them. Each task has a name, used for tracing (should be UpperCamelCase).
/// FIXME(sammccall): pull out a scheduler options struct.
class TUScheduler {
public:
TUScheduler(const GlobalCompilationDatabase &CDB, unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
std::unique_ptr<ParsingCallbacks> ASTCallbacks,
std::chrono::steady_clock::duration UpdateDebounce,
ASTRetentionPolicy RetentionPolicy);
~TUScheduler();
/// Returns estimated memory usage for each of the currently open files.
/// The order of results is unspecified.
std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
/// Returns a list of files with ASTs currently stored in memory. This method
/// is not very reliable and is only used for test. E.g., the results will not
/// contain files that currently run something over their AST.
std::vector<Path> getFilesWithCachedAST() const;
/// Schedule an update for \p File.
/// The compile command in \p Inputs is ignored; worker queries CDB to get
/// the actual compile command.
/// If diagnostics are requested (Yes), and the context is cancelled
/// before they are prepared, they may be skipped if eventual-consistency
/// permits it (i.e. WantDiagnostics is downgraded to Auto).
/// Returns true if the file was not previously tracked.
bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
/// Remove \p File from the list of tracked files and schedule removal of its
/// resources. Pending diagnostics for closed files may not be delivered, even
/// if requested with WantDiags::Auto or WantDiags::Yes.
void remove(PathRef File);
/// Returns the current contents of the buffer for File, per last update().
/// The returned StringRef may be invalidated by any write to TUScheduler.
llvm::StringRef getContents(PathRef File) const;
/// Schedule an async task with no dependencies.
void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
/// Schedule an async read of the AST. \p Action will be called when AST is
/// ready. The AST passed to \p Action refers to the version of \p File
/// tracked at the time of the call, even if new updates are received before
/// \p Action is executed.
/// If an error occurs during processing, it is forwarded to the \p Action
/// callback.
/// If the context is cancelled before the AST is ready, the callback will
/// receive a CancelledError.
void runWithAST(llvm::StringRef Name, PathRef File,
Callback<InputsAndAST> Action);
/// Controls whether preamble reads wait for the preamble to be up-to-date.
enum PreambleConsistency {
/// The preamble is generated from the current version of the file.
/// If the content was recently updated, we will wait until we have a
/// preamble that reflects that update.
/// This is the slowest option, and may be delayed by other tasks.
Consistent,
/// The preamble may be generated from an older version of the file.
/// Reading from locations in the preamble may cause files to be re-read.
/// This gives callers two options:
/// - validate that the preamble is still valid, and only use it if so
/// - accept that the preamble contents may be outdated, and try to avoid
/// reading source code from headers.
/// This is the fastest option, usually a preamble is available immediately.
Stale,
/// Besides accepting stale preamble, this also allow preamble to be absent
/// (not ready or failed to build).
StaleOrAbsent,
};
/// Schedule an async read of the preamble.
/// If there's no up-to-date preamble, we follow the PreambleConsistency
/// policy.
/// If an error occurs, it is forwarded to the \p Action callback.
/// Context cancellation is ignored and should be handled by the Action.
/// (In practice, the Action is almost always executed immediately).
void runWithPreamble(llvm::StringRef Name, PathRef File,
PreambleConsistency Consistency,
Callback<InputsAndPreamble> Action);
/// Wait until there are no scheduled or running tasks.
/// Mostly useful for synchronizing tests.
bool blockUntilIdle(Deadline D) const;
private:
/// This class stores per-file data in the Files map.
struct FileData;
public:
/// Responsible for retaining and rebuilding idle ASTs. An implementation is
/// an LRU cache.
class ASTCache;
// The file being built/processed in the current thread. This is a hack in
// order to get the file name into the index implementations. Do not depend on
// this inside clangd.
// FIXME: remove this when there is proper index support via build system
// integration.
static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
private:
const GlobalCompilationDatabase &CDB;
const bool StorePreamblesInMemory;
std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
Semaphore Barrier;
llvm::StringMap<std::unique_ptr<FileData>> Files;
std::unique_ptr<ASTCache> IdleASTs;
// None when running tasks synchronously and non-None when running tasks
// asynchronously.
llvm::Optional<AsyncTaskRunner> PreambleTasks;
llvm::Optional<AsyncTaskRunner> WorkerThreads;
std::chrono::steady_clock::duration UpdateDebounce;
};
} // namespace clangd
} // namespace clang
#endif
|