[Support] Beef up and expose the response file parsing in llvm::cl

The plan is to use it for clang and lld.

Major behavior changes:
- We can now parse UTF-16 files that have a byte order mark.
- PR16209: Don't drop backslashes on the floor if they don't escape
  anything.

The actual parsing loop was based on code from Clang's driver.cpp,
although it's been rewritten to track its state with control flow rather
than state variables.

Reviewers: hans

Differential Revision: http://llvm-reviews.chandlerc.com/D1170

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@186587 91177308-0d34-0410-b5e6-96231b3b80d8

unittests/Support/CommandLineTest.cpp

@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Config/config.h"
 #include "gtest/gtest.h"
@@ -118,4 +119,27 @@ TEST(CommandLineTest, UseOptionCategory) {
                                                   "Category.";
 }
 
+class StrDupSaver : public cl::StringSaver {
+  const char *SaveString(const char *Str) LLVM_OVERRIDE {
+    return strdup(Str);
+  }
+};
+
+TEST(CommandLineTest, TokenizeGNUCommandLine) {
+  const char *Input = "foo\\ bar \"foo bar\" \'foo bar\' 'foo\\\\bar' "
+                      "foo\"bar\"baz C:\\src\\foo.cpp \"C:\\src\\foo.cpp\"";
+  const char *const Output[] = { "foo bar", "foo bar", "foo bar", "foo\\bar",
+                                 "foobarbaz", "C:\\src\\foo.cpp",
+                                 "C:\\src\\foo.cpp" };
+  SmallVector<const char *, 0> Actual;
+  StrDupSaver Saver;
+  cl::TokenizeGNUCommandLine(Input, Saver, Actual);
+  EXPECT_EQ(array_lengthof(Output), Actual.size());
+  for (unsigned I = 0, E = Actual.size(); I != E; ++I) {
+    if (I < array_lengthof(Output))
+      EXPECT_STREQ(Output[I], Actual[I]);
+    free(const_cast<char *>(Actual[I]));
+  }
+}
+
 }  // anonymous namespace