[dsymutil] Implement the BinaryHolder object and gain archive support.

This object is meant to own the ObjectFiles and their underlying
MemoryBuffer. It is basically the equivalent of an OwningBinary
except that it efficiently handles Archives. It is optimized for
efficiently providing mappings of members of the same archive when
they are opened successively (which is standard in Darwin debug
maps, objects from the same archive will be contiguous).

Of course, the BinaryHolder will also be used by the DWARF linker
once it is commited, but for now only the debug map parser uses it.

With this change, you can run llvm-dsymutil on your Darwin debug build
of clang and get a complete debug map for it.

Differential Revision: http://reviews.llvm.org/D6690

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@225207 91177308-0d34-0410-b5e6-96231b3b80d8

test/tools/dsymutil/Inputs/basic1.c

@@ -16,6 +16,9 @@
       clang basic1-lto.o basic2-lto.o basic3-lto.o -o basic-lto.macho.x86_64 -Wl,-object_path_lto,$PWD/basic-lto.macho.x86_64.o -Wl,-dead_strip
       rm basic1-lto.o basic2-lto.o basic3-lto.o
 
+   Archive compilation (after basic compilation):
+      ar -q libbasic.a basic2.macho.x86_64.o basic3.macho.x86_64.o
+      clang basic1.macho.x86_64.o -lbasic -o basic-archive.macho.x86_64 -Wl,-dead_strip -L.
 */
 
 int foo(int);

test/tools/dsymutil/debug-map-parsing.test

@@ -1,7 +1,10 @@
 RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic.macho.x86_64 | FileCheck %s
 RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic-lto.macho.x86_64 | FileCheck %s --check-prefix=CHECK-LTO
+RUN: llvm-dsymutil -v -parse-only -oso-prepend-path=%p %p/Inputs/basic-archive.macho.x86_64 | FileCheck %s --check-prefix=CHECK-ARCHIVE
 RUN: llvm-dsymutil -v -parse-only %p/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=NOT-FOUND
 RUN: not llvm-dsymutil -v -parse-only %p/Inputs/inexistant 2>&1 | FileCheck %s --check-prefix=NO-EXECUTABLE
+
+
 Check that We can parse the debug map of the basic executable.
 
 CHECK-NOT: error
@@ -33,6 +36,32 @@ CHECK-LTO: 	00000000000008e8 => 0000000100001008	_private_int
 CHECK-LTO: 	00000000000008ec => 0000000100001004	_val
 CHECK-LTO: END DEBUG MAP
 
+Check thet we correctly handle debug maps with archive members (including only
+opening the archive once if mulitple of its members are used).
+
+CHECK-ARCHIVE:      trying to open {{.*}}basic-archive.macho.x86_64'
+CHECK-ARCHIVE-NEXT: 	loaded file.
+CHECK-ARCHIVE-NEXT: trying to open {{.*}}/Inputs/basic1.macho.x86_64.o'
+CHECK-ARCHIVE-NEXT: 	loaded file.
+CHECK-ARCHIVE-NEXT: trying to open {{.*}}/libbasic.a(basic2.macho.x86_64.o)'
+CHECK-ARCHIVE-NEXT: 	opened new archive {{.*}}/libbasic.a'
+CHECK-ARCHIVE-NEXT: 	found member in current archive.
+CHECK-ARCHIVE-NEXT: trying to open {{.*}}/libbasic.a(basic3.macho.x86_64.o)'
+CHECK-ARCHIVE-NEXT: 	found member in current archive.
+CHECK-ARCHIVE: DEBUG MAP:   object addr =>  executable addr	symbol name
+CHECK-ARCHIVE: /Inputs/basic1.macho.x86_64.o:
+CHECK-ARCHIVE: 	0000000000000000 => 0000000100000ea0	_main
+CHECK-ARCHIVE: /Inputs/./libbasic.a(basic2.macho.x86_64.o):
+CHECK-ARCHIVE: 	0000000000000310 => 0000000100001000	_baz
+CHECK-ARCHIVE: 	0000000000000020 => 0000000100000ed0	_foo
+CHECK-ARCHIVE: 	0000000000000070 => 0000000100000f20	_inc
+CHECK-ARCHIVE: 	0000000000000560 => 0000000100001004	_private_int
+CHECK-ARCHIVE: /Inputs/./libbasic.a(basic3.macho.x86_64.o):
+CHECK-ARCHIVE: 	0000000000000020 => 0000000100000f40	_bar
+CHECK-ARCHIVE: 	0000000000000070 => 0000000100000f90	_inc
+CHECK-ARCHIVE: 	0000000000000004 => 0000000100001008	_val
+CHECK-ARCHIVE: END DEBUG MAP
+
 Check that we warn about missing object files (this presumes that the files aren't
 present in the machine's /Inputs/ folder, which should be a pretty safe bet).
 

tools/dsymutil/BinaryHolder.cpp

@@ -0,0 +1,111 @@
+//===-- BinaryHolder.cpp --------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This program is a utility that aims to be a dropin replacement for
+// Darwin's dsymutil.
+//
+//===----------------------------------------------------------------------===//
+
+#include "BinaryHolder.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace dsymutil {
+
+ErrorOr<MemoryBufferRef>
+BinaryHolder::GetMemoryBufferForFile(StringRef Filename) {
+  if (Verbose)
+    outs() << "trying to open '" << Filename << "'\n";
+
+  // Try that first as it doesn't involve any filesystem access.
+  if (auto ErrOrArchiveMember = GetArchiveMemberBuffer(Filename))
+    return *ErrOrArchiveMember;
+
+  // If the name ends with a closing paren, there is a huge chance
+  // it is an archive member specification.
+  if (Filename.endswith(")"))
+    if (auto ErrOrArchiveMember = MapArchiveAndGetMemberBuffer(Filename))
+      return *ErrOrArchiveMember;
+
+  // Otherwise, just try opening a standard file. If this is an
+  // archive member specifiaction and any of the above didn't handle it
+  // (either because the archive is not there anymore, or because the
+  // archive doesn't contain the requested member), this will still
+  // provide a sensible error message.
+  auto ErrOrFile = MemoryBuffer::getFileOrSTDIN(Filename);
+  if (auto Err = ErrOrFile.getError())
+    return Err;
+
+  if (Verbose)
+    outs() << "\tloaded file.\n";
+  CurrentArchive.reset();
+  CurrentMemoryBuffer = std::move(ErrOrFile.get());
+  return CurrentMemoryBuffer->getMemBufferRef();
+}
+
+ErrorOr<MemoryBufferRef>
+BinaryHolder::GetArchiveMemberBuffer(StringRef Filename) {
+  if (!CurrentArchive)
+    return make_error_code(errc::no_such_file_or_directory);
+
+  StringRef CurArchiveName = CurrentArchive->getFileName();
+  if (!Filename.startswith(Twine(CurArchiveName, "(").str()))
+    return make_error_code(errc::no_such_file_or_directory);
+
+  // Remove the archive name and the parens around the archive member name.
+  Filename = Filename.substr(CurArchiveName.size() + 1).drop_back();
+
+  for (const auto &Child : CurrentArchive->children()) {
+    if (auto NameOrErr = Child.getName())
+      if (*NameOrErr == Filename) {
+        if (Verbose)
+          outs() << "\tfound member in current archive.\n";
+        return Child.getMemoryBufferRef();
+      }
+  }
+
+  return make_error_code(errc::no_such_file_or_directory);
+}
+
+ErrorOr<MemoryBufferRef>
+BinaryHolder::MapArchiveAndGetMemberBuffer(StringRef Filename) {
+  StringRef ArchiveFilename = Filename.substr(0, Filename.find('('));
+
+  auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(ArchiveFilename);
+  if (auto Err = ErrOrBuff.getError())
+    return Err;
+
+  if (Verbose)
+    outs() << "\topened new archive '" << ArchiveFilename << "'\n";
+  auto ErrOrArchive = object::Archive::create((*ErrOrBuff)->getMemBufferRef());
+  if (auto Err = ErrOrArchive.getError())
+    return Err;
+
+  CurrentArchive = std::move(*ErrOrArchive);
+  CurrentMemoryBuffer = std::move(*ErrOrBuff);
+
+  return GetArchiveMemberBuffer(Filename);
+}
+
+ErrorOr<const object::ObjectFile &>
+BinaryHolder::GetObjectFile(StringRef Filename) {
+  auto ErrOrMemBufferRef = GetMemoryBufferForFile(Filename);
+  if (auto Err = ErrOrMemBufferRef.getError())
+    return Err;
+
+  auto ErrOrObjectFile =
+      object::ObjectFile::createObjectFile(*ErrOrMemBufferRef);
+  if (auto Err = ErrOrObjectFile.getError())
+    return Err;
+
+  CurrentObjectFile = std::move(*ErrOrObjectFile);
+  return *CurrentObjectFile;
+}
+}
+}

tools/dsymutil/BinaryHolder.h

@@ -0,0 +1,104 @@
+//===-- BinaryHolder.h - Utility class for accessing binaries -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This program is a utility that aims to be a dropin replacement for
+// Darwin's dsymutil.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H
+#define LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H
+
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/Error.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorOr.h"
+
+namespace llvm {
+namespace dsymutil {
+
+/// \brief The BinaryHolder class is responsible for creating and
+/// owning ObjectFile objects and their underlying MemoryBuffer. This
+/// is different from a simple OwningBinary in that it handles
+/// accessing to archive members.
+///
+/// As an optimization, this class will reuse an already mapped and
+/// parsed Archive object if 2 successive requests target the same
+/// archive file (Which is always the case in debug maps).
+/// Currently it only owns one memory buffer at any given time,
+/// meaning that a mapping request will invalidate the previous memory
+/// mapping.
+class BinaryHolder {
+  std::unique_ptr<object::Archive> CurrentArchive;
+  std::unique_ptr<MemoryBuffer> CurrentMemoryBuffer;
+  std::unique_ptr<object::ObjectFile> CurrentObjectFile;
+  bool Verbose;
+
+  /// \brief Get the MemoryBufferRef for the file specification in \p
+  /// Filename from the current archive.
+  ///
+  /// This function performs no system calls, it just looks up a
+  /// potential match for the given \p Filename in the currently
+  /// mapped archive if there is one.
+  ErrorOr<MemoryBufferRef> GetArchiveMemberBuffer(StringRef Filename);
+
+  /// \brief Interpret Filename as an archive member specification,
+  /// map the corresponding archive to memory and return the
+  /// MemoryBufferRef corresponding to the described member.
+  ErrorOr<MemoryBufferRef> MapArchiveAndGetMemberBuffer(StringRef Filename);
+
+  /// \brief Return the MemoryBufferRef that holds the memory
+  /// mapping for the given \p Filename. This function will try to
+  /// parse archive member specifications of the form
+  /// /path/to/archive.a(member.o).
+  ///
+  /// The returned MemoryBufferRef points to a buffer owned by this
+  /// object. The buffer is valid until the next call to
+  /// GetMemoryBufferForFile() on this object.
+  ErrorOr<MemoryBufferRef> GetMemoryBufferForFile(StringRef Filename);
+
+public:
+  BinaryHolder(bool Verbose) : Verbose(Verbose) {}
+
+  /// \brief Get the ObjectFile designated by the \p Filename. This
+  /// might be an archive member specification of the form
+  /// /path/to/archive.a(member.o).
+  ///
+  /// Calling this function invalidates the previous mapping owned by
+  /// the BinaryHolder.
+  ErrorOr<const object::ObjectFile &> GetObjectFile(StringRef Filename);
+
+  /// \brief Wraps GetObjectFile() to return a derived ObjectFile type.
+  template <typename ObjectFileType>
+  ErrorOr<const ObjectFileType &> GetFileAs(StringRef Filename) {
+    auto ErrOrObjFile = GetObjectFile(Filename);
+    if (auto Err = ErrOrObjFile.getError())
+      return Err;
+    if (const auto *Derived = dyn_cast<ObjectFileType>(CurrentObjectFile.get()))
+      return *Derived;
+    return make_error_code(object::object_error::invalid_file_type);
+  }
+
+  /// \brief Access the currently owned ObjectFile. As successfull
+  /// call to GetObjectFile() or GetFileAs() must have been performed
+  /// before calling this.
+  const object::ObjectFile &Get() {
+    assert(CurrentObjectFile);
+    return *CurrentObjectFile;
+  }
+
+  /// \brief Access to a derived version of the currently owned
+  /// ObjectFile. The conversion must be known to be valid.
+  template <typename ObjectFileType> const ObjectFileType &GetAs() {
+    return cast<ObjectFileType>(*CurrentObjectFile);
+  }
+};
+}
+}
+#endif

tools/dsymutil/CMakeLists.txt

@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_tool(llvm-dsymutil
   dsymutil.cpp
+  BinaryHolder.cpp
   DebugMap.cpp
   DwarfLinker.cpp
   MachODebugMapParser.cpp

tools/dsymutil/MachODebugMapParser.cpp

@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "BinaryHolder.h"
 #include "DebugMap.h"
 #include "dsymutil.h"
 #include "llvm/Object/MachO.h"
@@ -20,9 +21,10 @@ using namespace llvm::object;
 
 class MachODebugMapParser {
 public:
-  MachODebugMapParser(StringRef BinaryPath, StringRef PathPrefix = "")
+  MachODebugMapParser(StringRef BinaryPath, StringRef PathPrefix = "",
+                      bool Verbose = false)
       : BinaryPath(BinaryPath), PathPrefix(PathPrefix),
-        CurrentDebugMapObject(nullptr) {}
+        MainBinaryHolder(Verbose), CurrentObjectHolder(Verbose) {}
 
   /// \brief Parses and returns the DebugMap of the input binary.
   /// \returns an error in case the provided BinaryPath doesn't exist
@@ -33,16 +35,16 @@ private:
   std::string BinaryPath;
   std::string PathPrefix;
 
-  /// OwningBinary constructed from the BinaryPath.
-  object::OwningBinary<object::MachOObjectFile> MainOwningBinary;
+  /// Owns the MemoryBuffer for the main binary.
+  BinaryHolder MainBinaryHolder;
   /// Map of the binary symbol addresses.
   StringMap<uint64_t> MainBinarySymbolAddresses;
   StringRef MainBinaryStrings;
   /// The constructed DebugMap.
   std::unique_ptr<DebugMap> Result;
 
-  /// Handle to the currently processed object file.
-  object::OwningBinary<object::MachOObjectFile> CurrentObjectFile;
+  /// Owns the MemoryBuffer for the currently handled object file.
+  BinaryHolder CurrentObjectHolder;
   /// Map of the currently processed object file symbol addresses.
   StringMap<uint64_t> CurrentObjectAddresses;
   /// Element of the debug map corresponfing to the current object file.
@@ -66,26 +68,10 @@ private:
 static void Warning(const Twine &Msg) { errs() << "warning: " + Msg + "\n"; }
 }
 
-static ErrorOr<OwningBinary<MachOObjectFile>>
-createMachOBinary(StringRef File) {
-  auto MemBufOrErr = MemoryBuffer::getFile(File);
-  if (auto Error = MemBufOrErr.getError())
-    return Error;
-
-  MemoryBufferRef BufRef = (*MemBufOrErr)->getMemBufferRef();
-  auto MachOOrErr = ObjectFile::createMachOObjectFile(BufRef);
-  if (auto Error = MachOOrErr.getError())
-    return Error;
-
-  return OwningBinary<MachOObjectFile>(std::move(*MachOOrErr),
-                                       std::move(*MemBufOrErr));
-}
-
 /// Reset the parser state coresponding to the current object
 /// file. This is to be called after an object file is finished
 /// processing.
 void MachODebugMapParser::resetParserState() {
-  CurrentObjectFile = OwningBinary<object::MachOObjectFile>();
   CurrentObjectAddresses.clear();
   CurrentDebugMapObject = nullptr;
 }
@@ -99,14 +85,13 @@ void MachODebugMapParser::switchToNewDebugMapObject(StringRef Filename) {
   SmallString<80> Path(PathPrefix);
   sys::path::append(Path, Filename);
 
-  auto MachOOrError = createMachOBinary(Path);
+  auto MachOOrError = CurrentObjectHolder.GetFileAs<MachOObjectFile>(Path);
   if (auto Error = MachOOrError.getError()) {
     Warning(Twine("cannot open debug object \"") + Path.str() + "\": " +
             Error.message() + "\n");
     return;
   }
 
-  CurrentObjectFile = std::move(*MachOOrError);
   loadCurrentObjectFileSymbols();
   CurrentDebugMapObject = &Result->addDebugMapObject(Path);
 }
@@ -115,14 +100,13 @@ void MachODebugMapParser::switchToNewDebugMapObject(StringRef Filename) {
 /// successful iterates over the STAB entries. The real parsing is
 /// done in handleStabSymbolTableEntry.
 ErrorOr<std::unique_ptr<DebugMap>> MachODebugMapParser::parse() {
-  auto MainBinaryOrError = createMachOBinary(BinaryPath);
-  if (auto Error = MainBinaryOrError.getError())
+  auto MainBinOrError = MainBinaryHolder.GetFileAs<MachOObjectFile>(BinaryPath);
+  if (auto Error = MainBinOrError.getError())
     return Error;
 
-  MainOwningBinary = std::move(*MainBinaryOrError);
+  const MachOObjectFile &MainBinary = *MainBinOrError;
   loadMainBinarySymbols();
   Result = make_unique<DebugMap>();
-  const auto &MainBinary = *MainOwningBinary.getBinary();
   MainBinaryStrings = MainBinary.getStringTableData();
   for (const SymbolRef &Symbol : MainBinary.symbols()) {
     const DataRefImpl &DRI = Symbol.getRawDataRefImpl();
@@ -190,9 +174,8 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex,
 /// Load the current object file symbols into CurrentObjectAddresses.
 void MachODebugMapParser::loadCurrentObjectFileSymbols() {
   CurrentObjectAddresses.clear();
-  const auto &Binary = *CurrentObjectFile.getBinary();
 
-  for (auto Sym : Binary.symbols()) {
+  for (auto Sym : CurrentObjectHolder.Get().symbols()) {
     StringRef Name;
     uint64_t Addr;
     if (Sym.getAddress(Addr) || Addr == UnknownAddressOrSize ||
@@ -215,9 +198,9 @@ uint64_t MachODebugMapParser::getMainBinarySymbolAddress(StringRef Name) {
 /// Load the interesting main binary symbols' addresses into
 /// MainBinarySymbolAddresses.
 void MachODebugMapParser::loadMainBinarySymbols() {
-  const MachOObjectFile &Binary = *MainOwningBinary.getBinary();
-  section_iterator Section = Binary.section_end();
-  for (const auto &Sym : Binary.symbols()) {
+  const MachOObjectFile &MainBinary = MainBinaryHolder.GetAs<MachOObjectFile>();
+  section_iterator Section = MainBinary.section_end();
+  for (const auto &Sym : MainBinary.symbols()) {
     SymbolRef::Type Type;
     // Skip undefined and STAB entries.
     if (Sym.getType(Type) || (Type & SymbolRef::ST_Debug) ||
@@ -243,7 +226,7 @@ namespace dsymutil {
 llvm::ErrorOr<std::unique_ptr<DebugMap>> parseDebugMap(StringRef InputFile,
                                                        StringRef PrependPath,
                                                        bool Verbose) {
-  MachODebugMapParser Parser(InputFile, PrependPath);
+  MachODebugMapParser Parser(InputFile, PrependPath, Verbose);
   return Parser.parse();
 }
 }