reference, declarationdefinition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
    1
    2
    3
    4
    5
    6
    7
    8
    9
   10
   11
   12
   13
   14
   15
   16
   17
   18
   19
   20
   21
   22
   23
   24
   25
   26
   27
   28
   29
   30
   31
   32
   33
   34
   35
   36
   37
   38
   39
   40
   41
   42
   43
   44
   45
   46
   47
   48
   49
   50
   51
   52
   53
   54
   55
   56
   57
   58
   59
   60
   61
   62
   63
   64
   65
   66
   67
   68
   69
   70
   71
   72
   73
   74
   75
   76
   77
   78
   79
   80
   81
   82
   83
   84
   85
   86
   87
   88
   89
   90
   91
   92
   93
   94
   95
   96
   97
   98
   99
  100
  101
  102
  103
  104
  105
  106
  107
  108
  109
  110
  111
  112
  113
  114
  115
  116
  117
  118
  119
  120
  121
  122
  123
  124
  125
  126
  127
  128
  129
  130
  131
  132
  133
  134
  135
  136
  137
  138
  139
  140
  141
  142
  143
  144
  145
  146
  147
  148
  149
  150
  151
  152
  153
  154
  155
  156
  157
  158
  159
  160
  161
  162
  163
  164
  165
  166
  167
  168
  169
  170
  171
  172
  173
  174
  175
  176
  177
  178
  179
  180
  181
  182
  183
  184
  185
  186
  187
  188
  189
  190
  191
  192
  193
  194
  195
  196
  197
  198
  199
  200
  201
  202
  203
  204
  205
  206
  207
  208
  209
  210
  211
  212
  213
  214
  215
  216
  217
  218
  219
  220
  221
  222
  223
  224
  225
  226
  227
  228
  229
  230
  231
  232
#!/usr/bin/env python
#
# Given a previous good compile narrow down miscompiles.
# Expects two directories named "before" and "after" each containing a set of
# assembly or object files where the "after" version is assumed to be broken.
# You also have to provide a script called "link_test". It is called with a list
# of files which should be linked together and result tested. "link_test" should
# returns with exitcode 0 if the linking and testing succeeded.
#
# abtest.py operates by taking all files from the "before" directory and
# in each step replacing one of them with a file from the "bad" directory.
#
# Additionally you can perform the same steps with a single .s file. In this
# mode functions are identified by " -- Begin function FunctionName" and
# " -- End function" markers. The abtest.py then takes all
# function from the file in the "before" directory and replaces one function
# with the corresponding function from the "bad" file in each step.
#
# Example usage to identify miscompiled files:
#    1. Create a link_test script, make it executable. Simple Example:
#          clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"
#    2. Run the script to figure out which files are miscompiled:
#       > ./abtest.py 
#       somefile.s: ok
#       someotherfile.s: skipped: same content
#       anotherfile.s: failed: './link_test' exitcode != 0
#       ...
# Example usage to identify miscompiled functions inside a file:
#    3. Run the tests on a single file (assuming before/file.s and
#       after/file.s exist)
#       > ./abtest.py file.s
#       funcname1 [0/XX]: ok
#       funcname2 [1/XX]: ok
#       funcname3 [2/XX]: skipped: same content
#       funcname4 [3/XX]: failed: './link_test' exitcode != 0
#       ...
from fnmatch import filter
from sys import stderr
import argparse
import filecmp
import os
import subprocess
import sys

LINKTEST="./link_test"
ESCAPE="\033[%sm"
BOLD=ESCAPE % "1"
RED=ESCAPE % "31"
NORMAL=ESCAPE % "0"
FAILED=RED+"failed"+NORMAL

def find(dir, file_filter=None):
    files = [walkdir[0]+"/"+file for walkdir in os.walk(dir) for file in walkdir[2]]
    if file_filter != None:
        files = filter(files, file_filter)
    return files

def error(message):
    stderr.write("Error: %s\n" % (message,))

def warn(message):
    stderr.write("Warning: %s\n" % (message,))

def extract_functions(file):
    functions = []
    in_function = None
    for line in open(file):
        marker = line.find(" -- Begin function ")
        if marker != -1:
            if in_function != None:
                warn("Missing end of function %s" % (in_function,))
            funcname = line[marker + 19:-1]
            in_function = funcname
            text = line
            continue

        marker = line.find(" -- End function")
        if marker != -1:
            text += line
            functions.append( (in_function, text) )
            in_function = None
            continue

        if in_function != None:
            text += line
    return functions

def replace_function(file, function, replacement, dest):
    out = open(dest, "w")
    skip = False
    found = False
    in_function = None
    for line in open(file):
        marker = line.find(" -- Begin function ")
        if marker != -1:
            if in_function != None:
                warn("Missing end of function %s" % (in_function,))
            funcname = line[marker + 19:-1]
            in_function = funcname
            if in_function == function:
                out.write(replacement)
                skip = True
        else:
            marker = line.find(" -- End function")
            if marker != -1:
                in_function = None
                if skip:
                    skip = False
                    continue

        if not skip:
            out.write(line)

def announce_test(name):
    stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
    stderr.flush()

def announce_result(result, info):
    stderr.write(result)
    if info != "":
        stderr.write(": %s" % info)
    stderr.write("\n")
    stderr.flush()

def testrun(files):
    linkline="%s %s" % (LINKTEST, " ".join(files),)
    res = subprocess.call(linkline, shell=True)
    if res != 0:
        announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST)
        return False
    else:
        announce_result("ok", "")
        return True

def check_files():
    """Check files mode"""
    for i in range(0, len(NO_PREFIX)):
        f = NO_PREFIX[i]
        b=baddir+"/"+f
        if b not in BAD_FILES:
            warn("There is no corresponding file to '%s' in %s" \
                 % (gooddir+"/"+f, baddir))
            continue

        announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX)))

        # combine files (everything from good except f)
        testfiles=[]
        skip=False
        for c in NO_PREFIX:
            badfile = baddir+"/"+c
            goodfile = gooddir+"/"+c
            if c == f:
                testfiles.append(badfile)
                if filecmp.cmp(goodfile, badfile):
                    announce_result("skipped", "same content")
                    skip = True
                    break
            else:
                testfiles.append(goodfile)
        if skip:
            continue
        testrun(testfiles)

def check_functions_in_file(base, goodfile, badfile):
    functions = extract_functions(goodfile)
    if len(functions) == 0:
        warn("Couldn't find any function in %s, missing annotations?" % (goodfile,))
        return
    badfunctions = dict(extract_functions(badfile))
    if len(functions) == 0:
        warn("Couldn't find any function in %s, missing annotations?" % (badfile,))
        return

    COMBINED="/tmp/combined.s"
    i = 0
    for (func,func_text) in functions:
        announce_test(func + " [%s/%s]" % (i+1, len(functions)))
        i+=1
        if func not in badfunctions:
            warn("Function '%s' missing from bad file" % func)
            continue
        if badfunctions[func] == func_text:
            announce_result("skipped", "same content")
            continue
        replace_function(goodfile, func, badfunctions[func], COMBINED)
        testfiles=[]
        for c in NO_PREFIX:
            if c == base:
                testfiles.append(COMBINED)
                continue
            testfiles.append(gooddir + "/" + c)

        testrun(testfiles)

parser = argparse.ArgumentParser()
parser.add_argument('--a', dest='dir_a', default='before')
parser.add_argument('--b', dest='dir_b', default='after')
parser.add_argument('--insane', help='Skip sanity check', action='store_true')
parser.add_argument('file', metavar='file', nargs='?')
config = parser.parse_args()

gooddir=config.dir_a
baddir=config.dir_b

BAD_FILES=find(baddir, "*")
GOOD_FILES=find(gooddir, "*")
NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES])

# "Checking whether build environment is sane ..."
if not config.insane:
    announce_test("sanity check")
    if not os.access(LINKTEST, os.X_OK):
        error("Expect '%s' to be present and executable" % (LINKTEST,))
        exit(1)

    res = testrun(GOOD_FILES)
    if not res:
        # "build environment is grinning and holding a spatula. Guess not."
        linkline="%s %s" % (LINKTEST, " ".join(GOOD_FILES),)
        stderr.write("\n%s\n\n" % linkline)
        stderr.write("Returned with exitcode != 0\n")
        sys.exit(1)

if config.file is not None:
    # File exchange mode
    goodfile = gooddir+"/"+config.file
    badfile = baddir+"/"+config.file
    check_functions_in_file(config.file, goodfile, badfile)
else:
    # Function exchange mode
    check_files()