1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
| """
Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
See https://llvm.org/LICENSE.txt for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Provides a class to build Python test event data structures.
"""
from __future__ import print_function
from __future__ import absolute_import
# System modules
import inspect
import time
import traceback
# Third-party modules
# LLDB modules
from . import build_exception
class EventBuilder(object):
"""Helper class to build test result event dictionaries."""
BASE_DICTIONARY = None
# Test Event Types
TYPE_JOB_RESULT = "job_result"
TYPE_TEST_RESULT = "test_result"
TYPE_TEST_START = "test_start"
TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
TYPE_SESSION_TERMINATE = "terminate"
RESULT_TYPES = {TYPE_JOB_RESULT, TYPE_TEST_RESULT}
# Test/Job Status Tags
STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
STATUS_SUCCESS = "success"
STATUS_FAILURE = "failure"
STATUS_EXPECTED_FAILURE = "expected_failure"
STATUS_EXPECTED_TIMEOUT = "expected_timeout"
STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
STATUS_SKIP = "skip"
STATUS_ERROR = "error"
STATUS_TIMEOUT = "timeout"
"""Test methods or jobs with a status matching any of these
status values will cause a testrun failure, unless
the test methods rerun and do not trigger an issue when rerun."""
TESTRUN_ERROR_STATUS_VALUES = {
STATUS_ERROR,
STATUS_EXCEPTIONAL_EXIT,
STATUS_FAILURE,
STATUS_TIMEOUT}
@staticmethod
def _get_test_name_info(test):
"""Returns (test-class-name, test-method-name) from a test case instance.
@param test a unittest.TestCase instance.
@return tuple containing (test class name, test method name)
"""
test_class_components = test.id().split(".")
test_class_name = ".".join(test_class_components[:-1])
test_name = test_class_components[-1]
return test_class_name, test_name
@staticmethod
def bare_event(event_type):
"""Creates an event with default additions, event type and timestamp.
@param event_type the value set for the "event" key, used
to distinguish events.
@returns an event dictionary with all default additions, the "event"
key set to the passed in event_type, and the event_time value set to
time.time().
"""
if EventBuilder.BASE_DICTIONARY is not None:
# Start with a copy of the "always include" entries.
event = dict(EventBuilder.BASE_DICTIONARY)
else:
event = {}
event.update({
"event": event_type,
"event_time": time.time()
})
return event
@staticmethod
def _assert_is_python_sourcefile(test_filename):
if test_filename is not None:
if not test_filename.endswith(".py"):
raise Exception(
"source python filename has unexpected extension: {}".format(test_filename))
return test_filename
@staticmethod
def _event_dictionary_common(test, event_type):
"""Returns an event dictionary setup with values for the given event type.
@param test the unittest.TestCase instance
@param event_type the name of the event type (string).
@return event dictionary with common event fields set.
"""
test_class_name, test_name = EventBuilder._get_test_name_info(test)
# Determine the filename for the test case. If there is an attribute
# for it, use it. Otherwise, determine from the TestCase class path.
if hasattr(test, "test_filename"):
test_filename = EventBuilder._assert_is_python_sourcefile(
test.test_filename)
else:
test_filename = EventBuilder._assert_is_python_sourcefile(
inspect.getsourcefile(test.__class__))
event = EventBuilder.bare_event(event_type)
event.update({
"test_class": test_class_name,
"test_name": test_name,
"test_filename": test_filename
})
return event
@staticmethod
def _error_tuple_class(error_tuple):
"""Returns the unittest error tuple's error class as a string.
@param error_tuple the error tuple provided by the test framework.
@return the error type (typically an exception) raised by the
test framework.
"""
type_var = error_tuple[0]
module = inspect.getmodule(type_var)
if module:
return "{}.{}".format(module.__name__, type_var.__name__)
else:
return type_var.__name__
@staticmethod
def _error_tuple_message(error_tuple):
"""Returns the unittest error tuple's error message.
@param error_tuple the error tuple provided by the test framework.
@return the error message provided by the test framework.
"""
return str(error_tuple[1])
@staticmethod
def _error_tuple_traceback(error_tuple):
"""Returns the unittest error tuple's error message.
@param error_tuple the error tuple provided by the test framework.
@return the error message provided by the test framework.
"""
return error_tuple[2]
@staticmethod
def _event_dictionary_test_result(test, status):
"""Returns an event dictionary with common test result fields set.
@param test a unittest.TestCase instance.
@param status the status/result of the test
(e.g. "success", "failure", etc.)
@return the event dictionary
"""
event = EventBuilder._event_dictionary_common(
test, EventBuilder.TYPE_TEST_RESULT)
event["status"] = status
return event
@staticmethod
def _event_dictionary_issue(test, status, error_tuple):
"""Returns an event dictionary with common issue-containing test result
fields set.
@param test a unittest.TestCase instance.
@param status the status/result of the test
(e.g. "success", "failure", etc.)
@param error_tuple the error tuple as reported by the test runner.
This is of the form (type<error>, error).
@return the event dictionary
"""
event = EventBuilder._event_dictionary_test_result(test, status)
event["issue_class"] = EventBuilder._error_tuple_class(error_tuple)
event["issue_message"] = EventBuilder._error_tuple_message(error_tuple)
backtrace = EventBuilder._error_tuple_traceback(error_tuple)
if backtrace is not None:
event["issue_backtrace"] = traceback.format_tb(backtrace)
return event
@staticmethod
def event_for_start(test):
"""Returns an event dictionary for the test start event.
@param test a unittest.TestCase instance.
@return the event dictionary
"""
return EventBuilder._event_dictionary_common(
test, EventBuilder.TYPE_TEST_START)
@staticmethod
def event_for_success(test):
"""Returns an event dictionary for a successful test.
@param test a unittest.TestCase instance.
@return the event dictionary
"""
return EventBuilder._event_dictionary_test_result(
test, EventBuilder.STATUS_SUCCESS)
@staticmethod
def event_for_unexpected_success(test, bugnumber):
"""Returns an event dictionary for a test that succeeded but was
expected to fail.
@param test a unittest.TestCase instance.
@param bugnumber the issue identifier for the bug tracking the
fix request for the test expected to fail (but is in fact
passing here).
@return the event dictionary
"""
event = EventBuilder._event_dictionary_test_result(
test, EventBuilder.STATUS_UNEXPECTED_SUCCESS)
if bugnumber:
event["bugnumber"] = str(bugnumber)
return event
@staticmethod
def event_for_failure(test, error_tuple):
"""Returns an event dictionary for a test that failed.
@param test a unittest.TestCase instance.
@param error_tuple the error tuple as reported by the test runner.
This is of the form (type<error>, error).
@return the event dictionary
"""
return EventBuilder._event_dictionary_issue(
test, EventBuilder.STATUS_FAILURE, error_tuple)
@staticmethod
def event_for_expected_failure(test, error_tuple, bugnumber):
"""Returns an event dictionary for a test that failed as expected.
@param test a unittest.TestCase instance.
@param error_tuple the error tuple as reported by the test runner.
This is of the form (type<error>, error).
@param bugnumber the issue identifier for the bug tracking the
fix request for the test expected to fail.
@return the event dictionary
"""
event = EventBuilder._event_dictionary_issue(
test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple)
if bugnumber:
event["bugnumber"] = str(bugnumber)
return event
@staticmethod
def event_for_skip(test, reason):
"""Returns an event dictionary for a test that was skipped.
@param test a unittest.TestCase instance.
@param reason the reason why the test is being skipped.
@return the event dictionary
"""
event = EventBuilder._event_dictionary_test_result(
test, EventBuilder.STATUS_SKIP)
event["skip_reason"] = reason
return event
@staticmethod
def event_for_error(test, error_tuple):
"""Returns an event dictionary for a test that hit a test execution error.
@param test a unittest.TestCase instance.
@param error_tuple the error tuple as reported by the test runner.
This is of the form (type<error>, error).
@return the event dictionary
"""
event = EventBuilder._event_dictionary_issue(
test, EventBuilder.STATUS_ERROR, error_tuple)
event["issue_phase"] = "test"
return event
@staticmethod
def event_for_build_error(test, error_tuple):
"""Returns an event dictionary for a test that hit a test execution error
during the test cleanup phase.
@param test a unittest.TestCase instance.
@param error_tuple the error tuple as reported by the test runner.
This is of the form (type<error>, error).
@return the event dictionary
"""
event = EventBuilder._event_dictionary_issue(
test, EventBuilder.STATUS_ERROR, error_tuple)
event["issue_phase"] = "build"
build_error = error_tuple[1]
event["build_command"] = build_error.command
event["build_error"] = build_error.build_error
return event
@staticmethod
def event_for_cleanup_error(test, error_tuple):
"""Returns an event dictionary for a test that hit a test execution error
during the test cleanup phase.
@param test a unittest.TestCase instance.
@param error_tuple the error tuple as reported by the test runner.
This is of the form (type<error>, error).
@return the event dictionary
"""
event = EventBuilder._event_dictionary_issue(
test, EventBuilder.STATUS_ERROR, error_tuple)
event["issue_phase"] = "cleanup"
return event
@staticmethod
def event_for_job_test_add_error(test_filename, exception, backtrace):
event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
event["status"] = EventBuilder.STATUS_ERROR
if test_filename is not None:
event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
test_filename)
if exception is not None and "__class__" in dir(exception):
event["issue_class"] = exception.__class__
event["issue_message"] = exception
if backtrace is not None:
event["issue_backtrace"] = backtrace
return event
@staticmethod
def event_for_job_exceptional_exit(
pid, worker_index, exception_code, exception_description,
test_filename, command_line):
"""Creates an event for a job (i.e. process) exit due to signal.
@param pid the process id for the job that failed
@param worker_index optional id for the job queue running the process
@param exception_code optional code
(e.g. SIGTERM integer signal number)
@param exception_description optional string containing symbolic
representation of the issue (e.g. "SIGTERM")
@param test_filename the path to the test filename that exited
in some exceptional way.
@param command_line the Popen()-style list provided as the command line
for the process that timed out.
@return an event dictionary coding the job completion description.
"""
event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
event["status"] = EventBuilder.STATUS_EXCEPTIONAL_EXIT
if pid is not None:
event["pid"] = pid
if worker_index is not None:
event["worker_index"] = int(worker_index)
if exception_code is not None:
event["exception_code"] = exception_code
if exception_description is not None:
event["exception_description"] = exception_description
if test_filename is not None:
event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
test_filename)
if command_line is not None:
event["command_line"] = command_line
return event
@staticmethod
def event_for_job_timeout(pid, worker_index, test_filename, command_line):
"""Creates an event for a job (i.e. process) timeout.
@param pid the process id for the job that timed out
@param worker_index optional id for the job queue running the process
@param test_filename the path to the test filename that timed out.
@param command_line the Popen-style list provided as the command line
for the process that timed out.
@return an event dictionary coding the job completion description.
"""
event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
event["status"] = "timeout"
if pid is not None:
event["pid"] = pid
if worker_index is not None:
event["worker_index"] = int(worker_index)
if test_filename is not None:
event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
test_filename)
if command_line is not None:
event["command_line"] = command_line
return event
@staticmethod
def event_for_mark_test_rerun_eligible(test):
"""Creates an event that indicates the specified test is explicitly
eligible for rerun.
Note there is a mode that will enable test rerun eligibility at the
global level. These markings for explicit rerun eligibility are
intended for the mode of running where only explicitly re-runnable
tests are rerun upon hitting an issue.
@param test the TestCase instance to which this pertains.
@return an event that specifies the given test as being eligible to
be rerun.
"""
event = EventBuilder._event_dictionary_common(
test,
EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE)
return event
@staticmethod
def event_for_mark_test_expected_failure(test):
"""Creates an event that indicates the specified test is expected
to fail.
@param test the TestCase instance to which this pertains.
@return an event that specifies the given test is expected to fail.
"""
event = EventBuilder._event_dictionary_common(
test,
EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
return event
@staticmethod
def add_entries_to_all_events(entries_dict):
"""Specifies a dictionary of entries to add to all test events.
This provides a mechanism for, say, a parallel test runner to
indicate to each inferior dotest.py that it should add a
worker index to each.
Calling this method replaces all previous entries added
by a prior call to this.
Event build methods will overwrite any entries that collide.
Thus, the passed in dictionary is the base, which gets merged
over by event building when keys collide.
@param entries_dict a dictionary containing key and value
pairs that should be merged into all events created by the
event generator. May be None to clear out any extra entries.
"""
EventBuilder.BASE_DICTIONARY = dict(entries_dict)
|