YAMLIO: Allow scalars to dictate quotation rules

Introduce ScalarTraits::mustQuote which determines whether or not a
StringRef needs quoting before it is acceptable to output.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@205955 91177308-0d34-0410-b5e6-96231b3b80d8

docs/YamlIO.rst

@@ -426,8 +426,10 @@ looks like:
       static StringRef input(StringRef scalar, T &value) {
         // do custom parsing here.  Return the empty string on success,
         // or an error message on failure.
-        return StringRef(); 
+        return StringRef();
       }
+      // Determine if this scalar needs quotes.
+      static bool mustQuote(StringRef) { return true; }
     };
     
 

include/llvm/Object/YAML.h

@@ -108,6 +108,7 @@ template <> struct ScalarTraits<object::yaml::BinaryRef> {
   static void output(const object::yaml::BinaryRef &, void *,
                      llvm::raw_ostream &);
   static StringRef input(StringRef, void *, object::yaml::BinaryRef &);
+  static bool mustQuote(StringRef S) { return needsQuotes(S); }
 };
 }
 

include/llvm/Support/YAMLTraits.h

@@ -20,6 +20,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/Regex.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/raw_ostream.h"
@@ -98,6 +99,7 @@ struct ScalarBitSetTraits {
 ///        // return empty string on success, or error string
 ///        return StringRef();
 ///      }
+///      static bool mustQuote(StringRef) { return true; }
 ///    };
 template<typename T>
 struct ScalarTraits {
@@ -109,6 +111,9 @@ struct ScalarTraits {
   // Function to convert a string to a value.  Returns the empty
   // StringRef on success or an error string if string is malformed:
   //static StringRef input(StringRef scalar, void *ctxt, T &value);
+  //
+  // Function to determine if the value should be quoted.
+  //static bool mustQuote(StringRef);
 };
 
 
@@ -198,17 +203,19 @@ struct has_ScalarTraits
 {
   typedef StringRef (*Signature_input)(StringRef, void*, T&);
   typedef void (*Signature_output)(const T&, void*, llvm::raw_ostream&);
+  typedef bool (*Signature_mustQuote)(StringRef);
 
   template <typename U>
-  static char test(SameType<Signature_input, &U::input>*,
+  static char test(SameType<Signature_input, &U::input> *,
-                   SameType<Signature_output, &U::output>*);
+                   SameType<Signature_output, &U::output> *,
+                   SameType<Signature_mustQuote, &U::mustQuote> *);
 
   template <typename U>
   static double test(...);
 
 public:
   static bool const value =
-    (sizeof(test<ScalarTraits<T> >(nullptr,nullptr)) == 1);
+      (sizeof(test<ScalarTraits<T>>(nullptr, nullptr, nullptr)) == 1);
 };
 
 
@@ -316,7 +323,81 @@ public:
   static bool const value =  (sizeof(test<DocumentListTraits<T> >(0)) == 1);
 };
 
+inline bool isNumber(StringRef S) {
+  static const char OctalChars[] = "01234567";
+  if (S.startswith("0") &&
+      S.drop_front().find_first_not_of(OctalChars) == StringRef::npos)
+    return true;
 
+  if (S.startswith("0o") &&
+      S.drop_front(2).find_first_not_of(OctalChars) == StringRef::npos)
+    return true;
+
+  static const char HexChars[] = "0123456789abcdefABCDEF";
+  if (S.startswith("0x") &&
+      S.drop_front(2).find_first_not_of(HexChars) == StringRef::npos)
+    return true;
+
+  static const char DecChars[] = "0123456789";
+  if (S.find_first_not_of(DecChars) == StringRef::npos)
+    return true;
+
+  if (S.equals(".inf") || S.equals(".Inf") || S.equals(".INF"))
+    return true;
+
+  Regex FloatMatcher("^(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?$");
+  if (FloatMatcher.match(S))
+    return true;
+
+  return false;
+}
+
+inline bool isNumeric(StringRef S) {
+  if ((S.front() == '-' || S.front() == '+') && isNumber(S.drop_front()))
+    return true;
+
+  if (isNumber(S))
+    return true;
+
+  if (S.equals(".nan") || S.equals(".NaN") || S.equals(".NAN"))
+    return true;
+
+  return false;
+}
+
+inline bool isNull(StringRef S) {
+  return S.equals("null") || S.equals("Null") || S.equals("NULL") ||
+         S.equals("~");
+}
+
+inline bool isBool(StringRef S) {
+  return S.equals("true") || S.equals("True") || S.equals("TRUE") ||
+         S.equals("false") || S.equals("False") || S.equals("FALSE");
+}
+
+inline bool needsQuotes(StringRef S) {
+  if (S.empty())
+    return true;
+  if (isspace(S.front()) || isspace(S.back()))
+    return true;
+  if (S.front() == ',')
+    return true;
+
+  static const char ScalarSafeChars[] =
+      "abcdefghijklmnopqrstuvwxyz"
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-/^., \t";
+  if (S.find_first_not_of(ScalarSafeChars) != StringRef::npos)
+    return true;
+
+  if (isNull(S))
+    return true;
+  if (isBool(S))
+    return true;
+  if (isNumeric(S))
+    return true;
+
+  return false;
+}
 
 
 template<typename T>
@@ -371,7 +452,7 @@ public:
   virtual bool bitSetMatch(const char*, bool) = 0;
   virtual void endBitSetScalar() = 0;
 
-  virtual void scalarString(StringRef &) = 0;
+  virtual void scalarString(StringRef &, bool) = 0;
 
   virtual void setError(const Twine &) = 0;
 
@@ -521,11 +602,11 @@ yamlize(IO &io, T &Val, bool) {
     llvm::raw_string_ostream Buffer(Storage);
     ScalarTraits<T>::output(Val, io.getContext(), Buffer);
     StringRef Str = Buffer.str();
-    io.scalarString(Str);
+    io.scalarString(Str, ScalarTraits<T>::mustQuote(Str));
   }
   else {
     StringRef Str;
-    io.scalarString(Str);
+    io.scalarString(Str, ScalarTraits<T>::mustQuote(Str));
     StringRef Result = ScalarTraits<T>::input(Str, io.getContext(), Val);
     if ( !Result.empty() ) {
       io.setError(llvm::Twine(Result));
@@ -602,78 +683,91 @@ template<>
 struct ScalarTraits<bool> {
   static void output(const bool &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, bool &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<StringRef> {
   static void output(const StringRef &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, StringRef &);
+  static bool mustQuote(StringRef S) { return needsQuotes(S); }
 };
  
 template<>
 struct ScalarTraits<std::string> {
   static void output(const std::string &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, std::string &);
+  static bool mustQuote(StringRef S) { return needsQuotes(S); }
 };
 
 template<>
 struct ScalarTraits<uint8_t> {
   static void output(const uint8_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, uint8_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<uint16_t> {
   static void output(const uint16_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, uint16_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<uint32_t> {
   static void output(const uint32_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, uint32_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<uint64_t> {
   static void output(const uint64_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, uint64_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<int8_t> {
   static void output(const int8_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, int8_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<int16_t> {
   static void output(const int16_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, int16_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<int32_t> {
   static void output(const int32_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, int32_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<int64_t> {
   static void output(const int64_t &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, int64_t &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<float> {
   static void output(const float &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, float &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<double> {
   static void output(const double &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, double &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 
@@ -795,7 +889,7 @@ private:
   bool beginBitSetScalar(bool &) override;
   bool bitSetMatch(const char *, bool ) override;
   void endBitSetScalar() override;
-  void scalarString(StringRef &) override;
+  void scalarString(StringRef &, bool) override;
   void setError(const Twine &message) override;
   bool canElideEmptySequence() override;
 
@@ -920,7 +1014,7 @@ public:
   bool beginBitSetScalar(bool &) override;
   bool bitSetMatch(const char *, bool ) override;
   void endBitSetScalar() override;
-  void scalarString(StringRef &) override;
+  void scalarString(StringRef &, bool) override;
   void setError(const Twine &message) override;
   bool canElideEmptySequence() override;
 public:
@@ -991,24 +1085,28 @@ template<>
 struct ScalarTraits<Hex8> {
   static void output(const Hex8 &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, Hex8 &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<Hex16> {
   static void output(const Hex16 &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, Hex16 &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<Hex32> {
   static void output(const Hex32 &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, Hex32 &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template<>
 struct ScalarTraits<Hex64> {
   static void output(const Hex64 &, void*, llvm::raw_ostream &);
   static StringRef input(StringRef, void*, Hex64 &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 

lib/MC/MCModuleYAML.cpp

@@ -162,12 +162,14 @@ template <> struct ScalarTraits<MCModuleYAML::Operand> {
   static void output(const MCModuleYAML::Operand &, void *,
                      llvm::raw_ostream &);
   static StringRef input(StringRef, void *, MCModuleYAML::Operand &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 template <> struct ScalarTraits<MCModuleYAML::OpcodeEnum> {
   static void output(const MCModuleYAML::OpcodeEnum &, void *,
                      llvm::raw_ostream &);
   static StringRef input(StringRef, void *, MCModuleYAML::OpcodeEnum &);
+  static bool mustQuote(StringRef) { return false; }
 };
 
 void ScalarEnumerationTraits<MCAtom::AtomKind>::enumeration(

lib/Support/YAMLTraits.cpp

@@ -285,7 +285,7 @@ void Input::endBitSetScalar() {
   }
 }
 
-void Input::scalarString(StringRef &S) {
+void Input::scalarString(StringRef &S, bool) {
   if (ScalarHNode *SN = dyn_cast<ScalarHNode>(CurrentNode)) {
     S = SN->value();
   } else {
@@ -541,10 +541,7 @@ void Output::endBitSetScalar() {
   this->outputUpToEndOfLine(" ]");
 }
 
-void Output::scalarString(StringRef &S) {
+void Output::scalarString(StringRef &S, bool MustQuote) {
-  const char ScalarSafeChars[] = "abcdefghijklmnopqrstuvwxyz"
-      "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-/^., \t";
-
   this->newLineCheck();
   if (S.empty()) {
     // Print '' for the empty string because leaving the field empty is not
@@ -552,11 +549,8 @@ void Output::scalarString(StringRef &S) {
     this->outputUpToEndOfLine("''");
     return;
   }
-  bool isOctalString = S.front() == '0' && S.size() > 2 && !S.startswith("0x");
+  if (!MustQuote) {
-  if (S.find_first_not_of(ScalarSafeChars) == StringRef::npos &&
+    // Only quote if we must.
-      !isspace(S.front()) && !isspace(S.back()) && !isOctalString) {
-    // If the string consists only of safe characters, print it out without
-    // quotes.
     this->outputUpToEndOfLine(S);
     return;
   }

test/Object/X86/objdump-cfg-invalid-opcode.yaml

@@ -38,7 +38,7 @@ Sections:
 #CFG:     Type:            Data
 
 ##    4:   06                      (bad)
-#CFG:     Content:         06
+#CFG:     Content:         '06'
 
 #CFG:   - StartAddress:    0x0000000000000005
 #CFG:     Size:            1

unittests/Support/YAMLIOTest.cpp

@@ -303,12 +303,22 @@ struct StringTypes {
   llvm::StringRef str4;
   llvm::StringRef str5;
   llvm::StringRef str6;
+  llvm::StringRef str7;
+  llvm::StringRef str8;
+  llvm::StringRef str9;
+  llvm::StringRef str10;
+  llvm::StringRef str11;
   std::string stdstr1;
   std::string stdstr2;
   std::string stdstr3;
   std::string stdstr4;
   std::string stdstr5;
   std::string stdstr6;
+  std::string stdstr7;
+  std::string stdstr8;
+  std::string stdstr9;
+  std::string stdstr10;
+  std::string stdstr11;
 };
 
 namespace llvm {
@@ -322,12 +332,22 @@ namespace yaml {
       io.mapRequired("str4",      st.str4);
       io.mapRequired("str5",      st.str5);
       io.mapRequired("str6",      st.str6);
+      io.mapRequired("str7",      st.str7);
+      io.mapRequired("str8",      st.str8);
+      io.mapRequired("str9",      st.str9);
+      io.mapRequired("str10",     st.str10);
+      io.mapRequired("str11",     st.str11);
       io.mapRequired("stdstr1",   st.stdstr1);
       io.mapRequired("stdstr2",   st.stdstr2);
       io.mapRequired("stdstr3",   st.stdstr3);
       io.mapRequired("stdstr4",   st.stdstr4);
       io.mapRequired("stdstr5",   st.stdstr5);
       io.mapRequired("stdstr6",   st.stdstr6);
+      io.mapRequired("stdstr7",   st.stdstr7);
+      io.mapRequired("stdstr8",   st.stdstr8);
+      io.mapRequired("stdstr9",   st.stdstr9);
+      io.mapRequired("stdstr10",  st.stdstr10);
+      io.mapRequired("stdstr11",  st.stdstr11);
     }
   };
 }
@@ -343,12 +363,22 @@ TEST(YAMLIO, TestReadWriteStringTypes) {
     map.str4 = "@ddd";
     map.str5 = "";
     map.str6 = "0000000004000000";
+    map.str7 = "true";
+    map.str8 = "FALSE";
+    map.str9 = "~";
+    map.str10 = "0.2e20";
+    map.str11 = "0x30";
     map.stdstr1 = "'eee";
     map.stdstr2 = "\"fff";
     map.stdstr3 = "`ggg";
     map.stdstr4 = "@hhh";
     map.stdstr5 = "";
     map.stdstr6 = "0000000004000000";
+    map.stdstr7 = "true";
+    map.stdstr8 = "FALSE";
+    map.stdstr9 = "~";
+    map.stdstr10 = "0.2e20";
+    map.stdstr11 = "0x30";
 
     llvm::raw_string_ostream ostr(intermediate);
     Output yout(ostr);
@@ -362,6 +392,11 @@ TEST(YAMLIO, TestReadWriteStringTypes) {
   EXPECT_NE(llvm::StringRef::npos, flowOut.find("'@ddd'"));
   EXPECT_NE(llvm::StringRef::npos, flowOut.find("''\n"));
   EXPECT_NE(llvm::StringRef::npos, flowOut.find("'0000000004000000'\n"));
+  EXPECT_NE(llvm::StringRef::npos, flowOut.find("'true'\n"));
+  EXPECT_NE(llvm::StringRef::npos, flowOut.find("'FALSE'\n"));
+  EXPECT_NE(llvm::StringRef::npos, flowOut.find("'~'\n"));
+  EXPECT_NE(llvm::StringRef::npos, flowOut.find("'0.2e20'\n"));
+  EXPECT_NE(llvm::StringRef::npos, flowOut.find("'0x30'\n"));
   EXPECT_NE(std::string::npos, flowOut.find("'''eee"));
   EXPECT_NE(std::string::npos, flowOut.find("'\"fff'"));
   EXPECT_NE(std::string::npos, flowOut.find("'`ggg'"));
@@ -612,6 +647,7 @@ namespace yaml {
           return "malformed by";
       }
     }
+    static bool mustQuote(StringRef) { return true; }
   };
 }
 }
@@ -673,6 +709,8 @@ namespace yaml {
       value = n;
       return StringRef();
     }
+
+    static bool mustQuote(StringRef) { return false; }
   };
 }
 }