Initial implementation of virtual file system

This adds the minimum virtual file system support to start migrating
FileManager onto the VFS.

Originally discussed here:
http://lists.cs.uiuc.edu/pipermail/cfe-dev/2014-February/035188.html

Differential Revision: http://llvm-reviews.chandlerc.com/D2745

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@201618 91177308-0d34-0410-b5e6-96231b3b80d8

Author: benlangmuir

Diff for: include/clang/Basic/FileManager.h

@@ -17,14 +17,14 @@
 
 #include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/VirtualFileSystem.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Allocator.h"
-#include "llvm/Support/FileSystem.h"
 // FIXME: Enhance libsystem to support inode and other fields in stat.
 #include <sys/types.h>
 
@@ -55,7 +55,7 @@ class DirectoryEntry {
 /// \brief Cached information about one file (either on disk
 /// or in the virtual file system).
 ///
-/// If the 'FD' member is valid, then this FileEntry has an open file
+/// If the 'File' member is valid, then this FileEntry has an open file
 /// descriptor for the file.
 class FileEntry {
   const char *Name;           // Name of the file.
@@ -67,31 +67,33 @@ class FileEntry {
   bool IsNamedPipe;
   bool InPCH;
 
-  /// FD - The file descriptor for the file entry if it is opened and owned
-  /// by the FileEntry.  If not, this is set to -1.
-  mutable int FD;
+  /// \brief The open file, if it is owned by the \p FileEntry.
+  mutable OwningPtr<vfs::File> File;
   friend class FileManager;
 
+  void closeFile() const {
+    File.reset(0); // rely on destructor to close File
+  }
+
 public:
   FileEntry(llvm::sys::fs::UniqueID UniqueID, bool IsNamedPipe, bool InPCH)
-      : Name(0), UniqueID(UniqueID), IsNamedPipe(IsNamedPipe), InPCH(InPCH),
-        FD(-1) {}
+      : Name(0), UniqueID(UniqueID), IsNamedPipe(IsNamedPipe), InPCH(InPCH)
+  {}
   // Add a default constructor for use with llvm::StringMap
   FileEntry()
-      : Name(0), UniqueID(0, 0), IsNamedPipe(false), InPCH(false), FD(-1) {}
+      : Name(0), UniqueID(0, 0), IsNamedPipe(false), InPCH(false)
+  {}
 
   FileEntry(const FileEntry &FE) {
     memcpy(this, &FE, sizeof(FE));
-    assert(FD == -1 && "Cannot copy a file-owning FileEntry");
+    assert(!File && "Cannot copy a file-owning FileEntry");
   }
 
   void operator=(const FileEntry &FE) {
     memcpy(this, &FE, sizeof(FE));
-    assert(FD == -1 && "Cannot assign a file-owning FileEntry");
+    assert(!File && "Cannot assign a file-owning FileEntry");
   }
 
-  ~FileEntry();
-
   const char *getName() const { return Name; }
   off_t getSize() const { return Size; }
   unsigned getUID() const { return UID; }
@@ -119,6 +121,7 @@ struct FileData;
 /// as a single file.
 ///
 class FileManager : public RefCountedBase<FileManager> {
+  IntrusiveRefCntPtr<vfs::FileSystem> FS;
   FileSystemOptions FileSystemOpts;
 
   class UniqueDirContainer;
@@ -172,14 +175,15 @@ class FileManager : public RefCountedBase<FileManager> {
   OwningPtr<FileSystemStatCache> StatCache;
 
   bool getStatValue(const char *Path, FileData &Data, bool isFile,
-                    int *FileDescriptor);
+                    vfs::File **F);
 
   /// Add all ancestors of the given path (pointing to either a file
   /// or a directory) as virtual directories.
   void addAncestorsAsVirtualDirs(StringRef Path);
 
 public:
-  FileManager(const FileSystemOptions &FileSystemOpts);
+  FileManager(const FileSystemOptions &FileSystemOpts,
+              llvm::IntrusiveRefCntPtr<vfs::FileSystem> FS = 0);
   ~FileManager();
 
   /// \brief Installs the provided FileSystemStatCache object within
@@ -248,7 +252,7 @@ class FileManager : public RefCountedBase<FileManager> {
   ///
   /// \returns false on success, true on error.
   bool getNoncachedStatValue(StringRef Path,
-                             llvm::sys::fs::file_status &Result);
+                             vfs::Status &Result);
 
   /// \brief Remove the real file \p Entry from the cache.
   void invalidateCache(const FileEntry *Entry);

Diff for: include/clang/Basic/FileSystemStatCache.h

@@ -24,6 +24,11 @@
 
 namespace clang {
 
+namespace vfs {
+class File;
+class FileSystem;
+}
+
 struct FileData {
   uint64_t Size;
   time_t ModTime;
@@ -57,10 +62,11 @@ class FileSystemStatCache {
   /// If isFile is true, then this lookup should only return success for files
   /// (not directories).  If it is false this lookup should only return
   /// success for directories (not files).  On a successful file lookup, the
-  /// implementation can optionally fill in FileDescriptor with a valid
-  /// descriptor and the client guarantees that it will close it.
+  /// implementation can optionally fill in \p F with a valid \p File object and
+  /// the client guarantees that it will close it.
   static bool get(const char *Path, FileData &Data, bool isFile,
-                  int *FileDescriptor, FileSystemStatCache *Cache);
+                  vfs::File **F, FileSystemStatCache *Cache,
+                  vfs::FileSystem &FS);
 
   /// \brief Sets the next stat call cache in the chain of stat caches.
   /// Takes ownership of the given stat cache.
@@ -78,17 +84,16 @@ class FileSystemStatCache {
 
 protected:
   virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                               int *FileDescriptor) = 0;
+                               vfs::File **F, vfs::FileSystem &FS) = 0;
 
   LookupResult statChained(const char *Path, FileData &Data, bool isFile,
-                           int *FileDescriptor) {
+                           vfs::File **F, vfs::FileSystem &FS) {
     if (FileSystemStatCache *Next = getNextStatCache())
-      return Next->getStat(Path, Data, isFile, FileDescriptor);
+      return Next->getStat(Path, Data, isFile, F, FS);
 
     // If we hit the end of the list of stat caches to try, just compute and
     // return it without a cache.
-    return get(Path, Data, isFile, FileDescriptor, 0) ? CacheMissing
-                                                      : CacheExists;
+    return get(Path, Data, isFile, F, 0, FS) ? CacheMissing : CacheExists;
   }
 };
 
@@ -107,7 +112,7 @@ class MemorizeStatCalls : public FileSystemStatCache {
   iterator end() const { return StatCalls.end(); }
 
   virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                               int *FileDescriptor);
+                               vfs::File **F, vfs::FileSystem &FS);
 };
 
 } // end namespace clang

Diff for: include/clang/Basic/VirtualFileSystem.h

@@ -0,0 +1,127 @@
+//===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// \brief Defines the virtual file system interface vfs::FileSystem.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H
+#define LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ErrorOr.h"
+
+namespace llvm {
+template <typename T> class OwningPtr;
+class MemoryBuffer;
+}
+
+namespace clang {
+namespace vfs {
+
+/// \brief The result of a \p status operation.
+class Status {
+  std::string Name;
+  std::string ExternalName;
+  llvm::sys::fs::UniqueID UID;
+  llvm::sys::TimeValue MTime;
+  uint32_t User;
+  uint32_t Group;
+  uint64_t Size;
+  llvm::sys::fs::file_type Type;
+  llvm::sys::fs::perms Perms;
+
+public:
+  Status() : Type(llvm::sys::fs::file_type::status_error) {}
+  Status(const llvm::sys::fs::file_status &Status);
+  Status(llvm::StringRef Name, llvm::StringRef RealName,
+         llvm::sys::fs::UniqueID UID, llvm::sys::TimeValue MTime,
+         uint32_t User, uint32_t Group, uint64_t Size,
+         llvm::sys::fs::file_type Type, llvm::sys::fs::perms Perms);
+
+  /// \brief Returns the name this status was looked up by.
+  llvm::StringRef getName() const { return Name; }
+
+  /// \brief Returns the name to use outside the compiler.
+  ///
+  /// For example, in diagnostics or debug info we should use this name.
+  llvm::StringRef getExternalName() const { return ExternalName; }
+
+  void setName(llvm::StringRef N) { Name = N; }
+  void setExternalName(llvm::StringRef N) { ExternalName = N; }
+
+  /// @name Status interface from llvm::sys::fs
+  /// @{
+  llvm::sys::fs::file_type getType() const { return Type; }
+  llvm::sys::fs::perms getPermissions() const { return Perms; }
+  llvm::sys::TimeValue getLastModificationTime() const { return MTime; }
+  llvm::sys::fs::UniqueID getUniqueID() const { return UID; }
+  uint32_t getUser() const { return User; }
+  uint32_t getGroup() const { return Group; }
+  uint64_t getSize() const { return Size; }
+  void setType(llvm::sys::fs::file_type v) { Type = v; }
+  void setPermissions(llvm::sys::fs::perms p) { Perms = p; }
+  /// @}
+  /// @name Status queries
+  /// These are static queries in llvm::sys::fs.
+  /// @{
+  bool equivalent(const Status &Other) const;
+  bool isDirectory() const;
+  bool isRegularFile() const;
+  bool isOther() const;
+  bool isSymlink() const;
+  bool isStatusKnown() const;
+  bool exists() const;
+  /// @}
+};
+
+/// \brief Represents an open file.
+class File {
+public:
+  /// \brief Destroy the file after closing it (if open).
+  /// Sub-classes should generally call close() inside their destructors.  We
+  /// cannot do that from the base class, since close is virtual.
+  virtual ~File();
+  /// \brief Get the status of the file.
+  virtual llvm::ErrorOr<Status> status() = 0;
+  /// \brief Get the contents of the file as a \p MemoryBuffer.
+  virtual llvm::error_code
+  getBuffer(const llvm::Twine &Name,
+            llvm::OwningPtr<llvm::MemoryBuffer> &Result, int64_t FileSize = -1,
+            bool RequiresNullTerminator = true) = 0;
+  /// \brief Closes the file.
+  virtual llvm::error_code close() = 0;
+};
+
+/// \brief The virtual file system interface.
+class FileSystem : public llvm::RefCountedBase<FileSystem> {
+public:
+  virtual ~FileSystem();
+
+  /// \brief Get the status of the entry at \p Path, if one exists.
+  virtual llvm::ErrorOr<Status> status(const llvm::Twine &Path) = 0;
+  /// \brief Get a \p File object for the file at \p Path, if one exists.
+  virtual llvm::error_code openFileForRead(const llvm::Twine &Path,
+                                           llvm::OwningPtr<File> &Result) = 0;
+
+  /// This is a convenience method that opens a file, gets its content and then
+  /// closes the file.
+  llvm::error_code getBufferForFile(const llvm::Twine &Name,
+                                    llvm::OwningPtr<llvm::MemoryBuffer> &Result,
+                                    int64_t FileSize = -1,
+                                    bool RequiresNullTerminator = true);
+};
+
+/// \brief Gets an \p vfs::FileSystem for the 'real' file system, as seen by
+/// the operating system.
+llvm::IntrusiveRefCntPtr<FileSystem> getRealFileSystem();
+
+} // end namespace vfs
+} // end namespace clang
+#endif // LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H

Diff for: include/clang/Frontend/CompilerInstance.h

@@ -75,6 +75,9 @@ class CompilerInstance : public ModuleLoader {
   /// The target being compiled for.
   IntrusiveRefCntPtr<TargetInfo> Target;
 
+  /// The virtual file system.
+  IntrusiveRefCntPtr<vfs::FileSystem> VirtualFileSystem;
+
   /// The file manager.
   IntrusiveRefCntPtr<FileManager> FileMgr;
 
@@ -313,6 +316,23 @@ class CompilerInstance : public ModuleLoader {
   /// Replace the current diagnostics engine.
   void setTarget(TargetInfo *Value);
 
+  /// }
+  /// @name Virtual File System
+  /// {
+
+  bool hasVirtualFileSystem() const { return VirtualFileSystem != 0; }
+
+  vfs::FileSystem &getVirtualFileSystem() const {
+    assert(hasVirtualFileSystem() &&
+           "Compiler instance has no virtual file system");
+    return *VirtualFileSystem;
+  }
+
+  /// \brief Replace the current virtual file system.
+  void setVirtualFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS) {
+    VirtualFileSystem = FS;
+  }
+
   /// }
   /// @name File Manager
   /// {
@@ -527,6 +547,10 @@ class CompilerInstance : public ModuleLoader {
                     bool ShouldOwnClient = true,
                     const CodeGenOptions *CodeGenOpts = 0);
 
+  /// Create a virtual file system and replace any existing one with it.
+  /// The default is to use the real file system.
+  void createVirtualFileSystem();
+
   /// Create the file manager and replace any existing one with it.
   void createFileManager();
 

Diff for: lib/Basic/CMakeLists.txt

@@ -23,6 +23,7 @@ add_clang_library(clangBasic
   TokenKinds.cpp
   Version.cpp
   VersionTuple.cpp
+  VirtualFileSystem.cpp
   )
 
 # Determine Subversion revision.