Generalize bugpoint's concept of a "safe" backend, and add options

to allow the "safe" backend to be run with a different path, and/or
with different command-line options.

This enables the following use cases:
 - bugpoint llc against an llc command from a different build
 - bugpoint llc against the same llc with different command-line options
 - and more...

Also, document the existing "custom" interpreter options.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@60681 91177308-0d34-0410-b5e6-96231b3b80d8

docs/CommandGuide/bugpoint.pod

@@ -57,6 +57,11 @@ The "--" right after the B<--tool-args> option tells B<bugpoint> to consider any
 options starting with C<-> to be part of the B<--tool-args> option, not as
 options to B<bugpoint> itself. (See B<--args>, above.)
 
+=item B<--safe-tool-args> I<tool args>
+
+Pass all arguments specified after --safe-tool-args to the "safe" execution
+tool.
+
 =item B<--disable-{dce,simplifycfg}>
 
 Do not run the specified passes to clean up and reduce the size of the test
@@ -103,18 +108,41 @@ to zero to disable the limit.
 Whenever the test program produces output on its standard output stream, it
 should match the contents of F<filename> (the "reference output"). If you
 do not use this option, B<bugpoint> will attempt to generate a reference output
-by compiling the program with the C backend and running it.
+by compiling the program with the "safe" backend and running it.
 
 =item B<--profile-info-file> F<filename>
 
 Profile file loaded by B<--profile-loader>.
 
-=item B<--run-{int,jit,llc,cbe}>
+=item B<--run-{int,jit,llc,cbe,custom}>
 
 Whenever the test program is compiled, B<bugpoint> should generate code for it
 using the specified code generator.  These options allow you to choose the
-interpreter, the JIT compiler, the static native code compiler, or the C
+interpreter, the JIT compiler, the static native code compiler, the C
-backend, respectively.
+backend, or a custom command (see B<--exec-command>) respectively.
+
+=item B<--safe-{llc,cbe,custom}>
+
+When debugging a code generator, B<bugpoint> should use the specified code
+generator as the "safe" code generator. This is a known-good code generator
+used to generate the "reference output" if it has not been provided, and to
+compile portions of the program that as they are excluded from the testcase.
+These options allow you to choose the
+static native code compiler, the C backend, or a custom command,
+(see B<--exec-command>) respectively. The interpreter and the JIT backends
+cannot currently be used as the "safe" backends.
+
+=item B<--exec-command> I<command>
+
+This option defines the command to use with the B<--run-custom> and
+B<--safe-custom> options to execute the bitcode testcase. This can
+be useful for cross-compilation.
+
+=item B<--safe-path> I<path>
+
+This option defines the path to the command to execute with the
+B<--safe-{int,jit,llc,cbe,custom}>
+option.
 
 =back
 

tools/bugpoint/BugDriver.cpp

@@ -65,7 +65,8 @@ std::string llvm::getPassesString(const std::vector<const PassInfo*> &Passes) {
 BugDriver::BugDriver(const char *toolname, bool as_child, bool find_bugs,
                      unsigned timeout, unsigned memlimit)
   : ToolName(toolname), ReferenceOutputFile(OutputFile),
-    Program(0), Interpreter(0), cbe(0), gcc(0), run_as_child(as_child),
+    Program(0), Interpreter(0), SafeInterpreter(0), gcc(0),
+    run_as_child(as_child),
     run_find_bugs(find_bugs), Timeout(timeout), MemoryLimit(memlimit) {}
 
 

tools/bugpoint/BugDriver.h

@@ -45,7 +45,7 @@ class BugDriver {
   Module *Program;             // The raw program, linked together
   std::vector<const PassInfo*> PassesToRun;
   AbstractInterpreter *Interpreter;   // How to run the program
-  AbstractInterpreter *cbe;
+  AbstractInterpreter *SafeInterpreter;  // To generate reference output, etc.
   GCC *gcc;
   bool run_as_child;
   bool run_find_bugs;
@@ -140,9 +140,9 @@ public:
     return OldProgram;
   }
 
-  AbstractInterpreter *switchToCBE() {
+  AbstractInterpreter *switchToSafeInterpreter() {
     AbstractInterpreter *Old = Interpreter;
-    Interpreter = (AbstractInterpreter*)cbe;
+    Interpreter = (AbstractInterpreter*)SafeInterpreter;
     return Old;
   }
 
@@ -172,11 +172,11 @@ public:
                              AbstractInterpreter *AI = 0,
                              bool *ProgramExitedNonzero = 0);
 
-  /// executeProgramWithCBE - Used to create reference output with the C
+  /// executeProgramSafely - Used to create reference output with the "safe"
   /// backend, if reference output is not provided.  If there is a problem with
   /// the code generator (e.g., llc crashes), this will throw an exception.
   ///
-  std::string executeProgramWithCBE(std::string OutputFile = "");
+  std::string executeProgramSafely(std::string OutputFile = "");
 
   /// createReferenceFile - calls compileProgram and then records the output
   /// into ReferenceOutputFile. Returns true if reference file created, false 

tools/bugpoint/ExecutionDriver.cpp

@@ -39,7 +39,7 @@ namespace {
                cl::init(0.0));
 
   cl::opt<OutputType>
-  InterpreterSel(cl::desc("Specify how LLVM code should be executed:"),
+  InterpreterSel(cl::desc("Specify the \"test\" i.e. suspect back-end:"),
                  cl::values(clEnumValN(AutoPick, "auto", "Use best guess"),
                             clEnumValN(RunLLI, "run-int",
                                        "Execute with the interpreter"),
@@ -54,6 +54,22 @@ namespace {
                             clEnumValEnd),
                  cl::init(AutoPick));
 
+  cl::opt<OutputType>
+  SafeInterpreterSel(cl::desc("Specify \"safe\" i.e. known-good backend:"),
+                     cl::values(clEnumValN(AutoPick, "safe-auto", "Use best guess"),
+                                clEnumValN(RunLLC, "safe-run-llc", "Compile with LLC"),
+                                clEnumValN(RunCBE, "safe-run-cbe", "Compile with CBE"),
+                                clEnumValN(Custom, "safe-run-custom",
+                                "Use -exec-command to define a command to execute "
+                                "the bitcode. Useful for cross-compilation."),
+                                clEnumValEnd),
+                     cl::init(AutoPick));
+
+  cl::opt<std::string>
+  SafeInterpreterPath("safe-path",
+                      cl::desc("Specify the path to the \"safe\" backend program"),
+                      cl::init(""));
+
   cl::opt<bool>
   AppendProgramExitCode("append-exit-code",
       cl::desc("Append the exit code to the output so it gets diff'd too"),
@@ -84,10 +100,17 @@ namespace llvm {
   cl::list<std::string>
   InputArgv("args", cl::Positional, cl::desc("<program arguments>..."),
             cl::ZeroOrMore, cl::PositionalEatsArgs);
+}
 
+namespace {
   cl::list<std::string>
   ToolArgv("tool-args", cl::Positional, cl::desc("<tool arguments>..."),
            cl::ZeroOrMore, cl::PositionalEatsArgs);
+
+  cl::list<std::string>
+  SafeToolArgv("safe-tool-args", cl::Positional,
+               cl::desc("<safe-tool arguments>..."),
+               cl::ZeroOrMore, cl::PositionalEatsArgs);
 }
 
 //===----------------------------------------------------------------------===//
@@ -102,14 +125,14 @@ bool BugDriver::initializeExecutionEnvironment() {
 
   // Create an instance of the AbstractInterpreter interface as specified on
   // the command line
-  cbe = 0;
+  SafeInterpreter = 0;
   std::string Message;
 
   switch (InterpreterSel) {
   case AutoPick:
     InterpreterSel = RunCBE;
-    Interpreter = cbe = AbstractInterpreter::createCBE(getToolName(), Message,
+    Interpreter =
-                                                       &ToolArgv);
+      AbstractInterpreter::createCBE(getToolName(), Message, &ToolArgv);
     if (!Interpreter) {
       InterpreterSel = RunJIT;
       Interpreter = AbstractInterpreter::createJIT(getToolName(), Message,
@@ -135,6 +158,7 @@ bool BugDriver::initializeExecutionEnvironment() {
                                                  &ToolArgv);
     break;
   case RunLLC:
+  case LLC_Safe:
     Interpreter = AbstractInterpreter::createLLC(getToolName(), Message,
                                                  &ToolArgv);
     break;
@@ -142,10 +166,6 @@ bool BugDriver::initializeExecutionEnvironment() {
     Interpreter = AbstractInterpreter::createJIT(getToolName(), Message,
                                                  &ToolArgv);
     break;
-  case LLC_Safe:
-    Interpreter = AbstractInterpreter::createLLC(getToolName(), Message,
-                                                 &ToolArgv);
-    break;
   case RunCBE:
   case CBE_bug:
     Interpreter = AbstractInterpreter::createCBE(getToolName(), Message,
@@ -164,20 +184,71 @@ bool BugDriver::initializeExecutionEnvironment() {
   else // Display informational messages on stdout instead of stderr
     std::cout << Message;
 
-  // Initialize auxiliary tools for debugging
+  std::string Path = SafeInterpreterPath;
-  if (InterpreterSel == RunCBE) {
+  if (Path.empty())
-    // We already created a CBE, reuse it.
+    Path = getToolName();
-    cbe = Interpreter;
+  std::vector<std::string> SafeToolArgs = SafeToolArgv;
-  } else if (InterpreterSel == CBE_bug || InterpreterSel == LLC_Safe) {
+  switch (SafeInterpreterSel) {
-    // We want to debug the CBE itself or LLC is known-good.  Use LLC as the
+  case AutoPick:
-    // 'known-good' compiler.
+    // In "cbe-bug" mode, default to using LLC as the "safe" backend.
-    std::vector<std::string> ToolArgs;
+    if (!SafeInterpreter &&
-    ToolArgs.push_back("--relocation-model=pic");
+        InterpreterSel == CBE_bug) {
-    cbe = AbstractInterpreter::createLLC(getToolName(), Message, &ToolArgs);
+      SafeInterpreterSel = RunLLC;
-  } else {
+      SafeToolArgs.push_back("--relocation-model=pic");
-    cbe = AbstractInterpreter::createCBE(getToolName(), Message, &ToolArgv);
+      SafeInterpreter = AbstractInterpreter::createLLC(Path, Message,
+                                                       &SafeToolArgs);
+    }
+
+    // In "llc-safe" mode, default to using LLC as the "safe" backend.
+    if (!SafeInterpreter &&
+        InterpreterSel == LLC_Safe) {
+      SafeInterpreterSel = RunLLC;
+      SafeToolArgs.push_back("--relocation-model=pic");
+      SafeInterpreter = AbstractInterpreter::createLLC(Path, Message,
+                                                       &SafeToolArgs);
+    }
+
+    // Pick a backend that's different from the test backend. The JIT and
+    // LLC backends share a lot of code, so prefer to use the CBE as the
+    // safe back-end when testing them.
+    if (!SafeInterpreter &&
+        InterpreterSel != RunCBE) {
+      SafeInterpreterSel = RunCBE;
+      SafeInterpreter = AbstractInterpreter::createCBE(Path, Message,
+                                                       &SafeToolArgs);
+    }
+    if (!SafeInterpreter &&
+        InterpreterSel != RunLLC &&
+        InterpreterSel != RunJIT) {
+      SafeInterpreterSel = RunLLC;
+      SafeToolArgs.push_back("--relocation-model=pic");
+      SafeInterpreter = AbstractInterpreter::createLLC(Path, Message,
+                                                       &SafeToolArgs);
+    }
+    if (!SafeInterpreter) {
+      SafeInterpreterSel = AutoPick;
+      Message = "Sorry, I can't automatically select an interpreter!\n";
+    }
+    break;
+  case RunLLC:
+    SafeToolArgs.push_back("--relocation-model=pic");
+    SafeInterpreter = AbstractInterpreter::createLLC(Path, Message,
+                                                     &SafeToolArgs);
+    break;
+  case RunCBE:
+    SafeInterpreter = AbstractInterpreter::createCBE(Path, Message,
+                                                     &SafeToolArgs);
+    break;
+  case Custom:
+    SafeInterpreter = AbstractInterpreter::createCustom(Path, Message,
+                                                        CustomExecCommand);
+    break;
+  default:
+    Message = "Sorry, this back-end is not supported by bugpoint as the "
+              "\"safe\" backend right now!\n";
+    break;
   }
-  if (!cbe) { std::cout << Message << "\nExiting.\n"; exit(1); }
+  if (!SafeInterpreter) { std::cout << Message << "\nExiting.\n"; exit(1); }
   
   gcc = GCC::create(getToolName(), Message);
   if (!gcc) { std::cout << Message << "\nExiting.\n"; exit(1); }
@@ -264,20 +335,9 @@ std::string BugDriver::executeProgram(std::string OutputFile,
   if (!SharedObj.empty())
     SharedObjs.push_back(SharedObj);
 
-  
+  int RetVal = AI->ExecuteProgram(BitcodeFile, InputArgv, InputFile,
-  // If this is an LLC or CBE run, then the GCC compiler might get run to 
+                                  OutputFile, AdditionalLinkerArgs, SharedObjs, 
-  // compile the program. If so, we should pass the user's -Xlinker options
+                                  Timeout, MemoryLimit);
-  // as the GCCArgs.
-  int RetVal = 0;
-  if (InterpreterSel == RunLLC || InterpreterSel == RunCBE ||
-      InterpreterSel == CBE_bug || InterpreterSel == LLC_Safe)
-    RetVal = AI->ExecuteProgram(BitcodeFile, InputArgv, InputFile,
-                                OutputFile, AdditionalLinkerArgs, SharedObjs, 
-                                Timeout, MemoryLimit);
-  else 
-    RetVal = AI->ExecuteProgram(BitcodeFile, InputArgv, InputFile,
-                                OutputFile, std::vector<std::string>(), 
-                                SharedObjs, Timeout, MemoryLimit);
 
   if (RetVal == -1) {
     std::cerr << "<timeout>";
@@ -305,12 +365,12 @@ std::string BugDriver::executeProgram(std::string OutputFile,
   return OutputFile;
 }
 
-/// executeProgramWithCBE - Used to create reference output with the C
+/// executeProgramSafely - Used to create reference output with the "safe"
 /// backend, if reference output is not provided.
 ///
-std::string BugDriver::executeProgramWithCBE(std::string OutputFile) {
+std::string BugDriver::executeProgramSafely(std::string OutputFile) {
   bool ProgramExitedNonzero;
-  std::string outFN = executeProgram(OutputFile, "", "", cbe,
+  std::string outFN = executeProgram(OutputFile, "", "", SafeInterpreter,
                                      &ProgramExitedNonzero);
   return outFN;
 }
@@ -319,8 +379,8 @@ std::string BugDriver::compileSharedObject(const std::string &BitcodeFile) {
   assert(Interpreter && "Interpreter should have been created already!");
   sys::Path OutputFile;
 
-  // Using CBE
+  // Using the known-good backend.
-  GCC::FileType FT = cbe->OutputCode(BitcodeFile, OutputFile);
+  GCC::FileType FT = SafeInterpreter->OutputCode(BitcodeFile, OutputFile);
 
   std::string SharedObjectFile;
   if (gcc->MakeSharedObject(OutputFile.toString(), FT,
@@ -345,14 +405,15 @@ bool BugDriver::createReferenceFile(Module *M, const std::string &Filename) {
     return false;
   }
   try {
-    ReferenceOutputFile = executeProgramWithCBE(Filename);
+    ReferenceOutputFile = executeProgramSafely(Filename);
     std::cout << "Reference output is: " << ReferenceOutputFile << "\n\n";
   } catch (ToolExecutionError &TEE) {
     std::cerr << TEE.what();
-    if (Interpreter != cbe) {
+    if (Interpreter != SafeInterpreter) {
-      std::cerr << "*** There is a bug running the C backend.  Either debug"
+      std::cerr << "*** There is a bug running the \"safe\" backend.  Either"
-                << " it (use the -run-cbe bugpoint option), or fix the error"
+                << " debug it (for example with the -run-cbe bugpoint option,"
-                << " some other way.\n";
+                << " if CBE is being used as the \"safe\" backend), or fix the"
+                << " error some other way.\n";
     }
     return false;
   }

tools/bugpoint/Miscompilation.cpp

@@ -277,7 +277,7 @@ static bool ExtractLoops(BugDriver &BD,
     // we're going to test the newly loop extracted program to make sure nothing
     // has broken.  If something broke, then we'll inform the user and stop
     // extraction.
-    AbstractInterpreter *AI = BD.switchToCBE();
+    AbstractInterpreter *AI = BD.switchToSafeInterpreter();
     if (TestMergedProgram(BD, ToOptimizeLoopExtracted, ToNotOptimize, false)) {
       BD.switchToInterpreter(AI);
 
@@ -838,13 +838,15 @@ static bool TestCodeGenerator(BugDriver &BD, Module *Test, Module *Safe) {
 /// debugCodeGenerator - debug errors in LLC, LLI, or CBE.
 ///
 bool BugDriver::debugCodeGenerator() {
-  if ((void*)cbe == (void*)Interpreter) {
+  if ((void*)SafeInterpreter == (void*)Interpreter) {
-    std::string Result = executeProgramWithCBE("bugpoint.cbe.out");
+    std::string Result = executeProgramSafely("bugpoint.safe.out");
-    std::cout << "\n*** The C backend cannot match the reference diff, but it "
+    std::cout << "\n*** The \"safe\" i.e. 'known good' backend cannot match "
-              << "is used as the\n    'known good' code generator, so I can't"
+              << "the reference diff.  This may be due to a\n    front-end "
-              << " debug it.  Perhaps you have a\n    front-end problem?  As a"
+              << "bug or a bug in the original program, but this can also "
-              << " sanity check, I left the result of executing the\n    "
+              << "happen if bugpoint isn't running the program with the "
-              << "program with the C backend in this file for you: '"
+              << "right flags or input.\n    I left the result of executing "
+              << "the program with the \"safe\" backend in this file for "
+              << "you: '"
               << Result << "'.\n";
     return true;
   }