[lib/Fuzzer] use -fsanitize-coverage=trace-cmp when building LLVM with LLVM_USE_SANITIZE_COVERAGE; in lib/Fuzzer try to reload the corpus to pick up new units from other processes

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@236906 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Kostya Serebryany 2015-05-08 21:30:55 +00:00
parent 262697d9d8
commit 8ae273d380
7 changed files with 69 additions and 11 deletions

View File

@ -476,7 +476,7 @@ if(LLVM_USE_SANITIZER)
message(WARNING "LLVM_USE_SANITIZER is not supported on this platform.")
endif()
if (LLVM_USE_SANITIZE_COVERAGE)
append("-fsanitize-coverage=edge,indirect-calls,8bit-counters" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
append("-fsanitize-coverage=edge,indirect-calls,8bit-counters,trace-cmp" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
endif()
endif()

View File

@ -207,6 +207,7 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
Options.PreferSmallDuringInitialShuffle =
Flags.prefer_small_during_initial_shuffle;
Options.Tokens = ReadTokensFile(Flags.tokens);
Options.Reload = Flags.reload;
if (Flags.runs >= 0)
Options.MaxNumberOfRuns = Flags.runs;
if (!inputs.empty())
@ -235,8 +236,10 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
if (Flags.apply_tokens)
return ApplyTokens(F, Flags.apply_tokens);
F.RereadOutputCorpus();
for (auto &inp : inputs)
F.ReadDir(inp);
if (inp != Options.OutputCorpus)
F.ReadDir(inp, nullptr);
if (F.CorpusSize() == 0)
F.AddToCorpus(Unit()); // Can't fuzz empty corpus, so add an empty input.

View File

@ -46,7 +46,9 @@ FUZZER_FLAG_INT(workers, 0,
"Number of simultaneous worker processes to run the jobs.")
FUZZER_FLAG_INT(dfsan, 1, "Use DFSan for taint-guided mutations. No-op unless "
"the DFSan instrumentation was compiled in.")
FUZZER_FLAG_INT(reload, 1,
"Reload the main corpus periodically to get new units"
"discovered by other processes.")
FUZZER_FLAG_STRING(tokens, "Use the file with tokens (one token per line) to"
" fuzz a token based input language.")
FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes "

View File

@ -13,10 +13,26 @@
#include <iterator>
#include <fstream>
#include <dirent.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
namespace fuzzer {
static std::vector<std::string> ListFilesInDir(const std::string &Dir) {
static long GetEpoch(const std::string &Path) {
struct stat St;
if (stat(Path.c_str(), &St)) return 0;
return St.st_mtime;
}
static std::vector<std::string> ListFilesInDir(const std::string &Dir,
long *Epoch) {
std::vector<std::string> V;
if (Epoch) {
auto E = GetEpoch(Dir.c_str());
if (*Epoch >= E) return V;
*Epoch = E;
}
DIR *D = opendir(Dir.c_str());
if (!D) return V;
while (auto E = readdir(D)) {
@ -50,9 +66,14 @@ void WriteToFile(const Unit &U, const std::string &Path) {
OF.write((const char*)U.data(), U.size());
}
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V) {
for (auto &X : ListFilesInDir(Path))
V->push_back(FileToVector(DirPlusFile(Path, X)));
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
long *Epoch) {
long E = Epoch ? *Epoch : 0;
for (auto &X : ListFilesInDir(Path, Epoch)) {
auto FilePath = DirPlusFile(Path, X);
if (Epoch && GetEpoch(FilePath) < E) continue;
V->push_back(FileToVector(FilePath));
}
}
std::string DirPlusFile(const std::string &DirPath,

View File

@ -16,6 +16,7 @@
#include <string>
#include <vector>
#include <unordered_set>
#include <set>
#include "FuzzerInterface.h"
@ -25,7 +26,8 @@ using namespace std::chrono;
std::string FileToString(const std::string &Path);
Unit FileToVector(const std::string &Path);
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V);
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
long *Epoch);
void WriteToFile(const Unit &U, const std::string &Path);
void CopyFileToErr(const std::string &Path);
// Returns "Dir/FileName" or equivalent for the current OS.
@ -54,6 +56,7 @@ class Fuzzer {
bool UseFullCoverageSet = false;
bool UseCoveragePairs = false;
bool UseDFSan = false;
bool Reload = true;
int PreferSmallDuringInitialShuffle = -1;
size_t MaxNumberOfRuns = ULONG_MAX;
std::string OutputCorpus;
@ -65,9 +68,10 @@ class Fuzzer {
void ShuffleAndMinimize();
void InitializeDFSan();
size_t CorpusSize() const { return Corpus.size(); }
void ReadDir(const std::string &Path) {
ReadDirToVectorOfUnits(Path.c_str(), &Corpus);
void ReadDir(const std::string &Path, long *Epoch) {
ReadDirToVectorOfUnits(Path.c_str(), &Corpus, Epoch);
}
void RereadOutputCorpus();
// Save the current corpus to OutputCorpus.
void SaveCorpus();
@ -116,6 +120,7 @@ class Fuzzer {
size_t TotalNumberOfRuns = 0;
std::vector<Unit> Corpus;
std::set<Unit> UnitsAddedAfterInitialLoad;
std::unordered_set<uintptr_t> FullCoverageSets;
std::unordered_set<uint64_t> CoveragePairs;
@ -132,6 +137,7 @@ class Fuzzer {
system_clock::time_point ProcessStartTime = system_clock::now();
system_clock::time_point UnitStartTime;
long TimeOfLongestUnitInSeconds = 0;
long EpochOfLastReadOfOutputCorpus = 0;
};
}; // namespace fuzzer

View File

@ -86,6 +86,30 @@ void Fuzzer::PrintStats(const char *Where, size_t Cov, const char *End) {
<< End;
}
void Fuzzer::RereadOutputCorpus() {
if (Options.OutputCorpus.empty()) return;
std::vector<Unit> AdditionalCorpus;
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus);
if (Corpus.empty()) {
Corpus = AdditionalCorpus;
return;
}
if (!Options.Reload) return;
for (auto &X : AdditionalCorpus) {
if (X.size() > (size_t)Options.MaxLen)
X.resize(Options.MaxLen);
if (UnitsAddedAfterInitialLoad.insert(X).second) {
Corpus.push_back(X);
CurrentUnit.clear();
CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
size_t NewCoverage = RunOne(CurrentUnit);
if (NewCoverage && Options.Verbosity >= 1)
PrintStats("RELOAD", NewCoverage);
}
}
}
void Fuzzer::ShuffleAndMinimize() {
size_t MaxCov = 0;
bool PreferSmall =
@ -268,6 +292,7 @@ void Fuzzer::SaveCorpus() {
void Fuzzer::ReportNewCoverage(size_t NewCoverage, const Unit &U) {
if (!NewCoverage) return;
Corpus.push_back(U);
UnitsAddedAfterInitialLoad.insert(U);
PrintStats("NEW ", NewCoverage, "");
if (Options.Verbosity) {
std::cerr << " L: " << U.size();
@ -299,6 +324,7 @@ void Fuzzer::MutateAndTestOne(Unit *U) {
void Fuzzer::Loop(size_t NumIterations) {
for (size_t i = 1; i <= NumIterations; i++) {
for (size_t J1 = 0; J1 < Corpus.size(); J1++) {
RereadOutputCorpus();
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
return;
// First, simply mutate the unit w/o doing crosses.

View File

@ -1,7 +1,7 @@
# These tests depend on both coverage and dfsan instrumentation.
set(CMAKE_CXX_FLAGS_RELEASE
"${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow -mllvm -sanitizer-coverage-experimental-trace-compares=1")
"${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow")
foreach(Test ${DFSanTests})
add_executable(LLVMFuzzer-${Test}