Support formatting of preprocessor branches.

We now correctly format:
  void SomeFunction(int param1,
  #ifdef X
                    NoTemplate param2,
  #else
                    template <
  #ifdef A
                        MyType<Some> >
  #else
                        Type1, Type2>
  #endif
                    param2,
  #endif
                    param3) {
    f();
  }

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@192503 91177308-0d34-0410-b5e6-96231b3b80d8

Author: r4nt

Diff for: lib/Format/ContinuationIndenter.cpp

@@ -221,7 +221,7 @@ unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
 
 void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
                                                  unsigned ExtraSpaces) {
-  const FormatToken &Current = *State.NextToken;
+  FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *State.NextToken->Previous;
   if (Current.is(tok::equal) &&
       (State.Line->First->is(tok::kw_for) || State.ParenLevel == 0) &&
@@ -301,7 +301,7 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
 
 unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
                                                  bool DryRun) {
-  const FormatToken &Current = *State.NextToken;
+  FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *State.NextToken->Previous;
   // If we are continuing an expression, we want to indent an extra 4 spaces.
   unsigned ContinuationIndent =

Diff for: lib/Format/ContinuationIndenter.h

@@ -250,7 +250,7 @@ struct LineState {
   unsigned Column;
 
   /// \brief The token that needs to be next formatted.
-  const FormatToken *NextToken;
+  FormatToken *NextToken;
 
   /// \brief \c true if this line contains a continued for-loop section.
   bool LineContainsContinuedForLoopSection;

Diff for: lib/Format/Format.cpp

@@ -447,14 +447,19 @@ class UnwrappedLineFormatter {
         // State already examined with lower penalty.
         continue;
 
-      addNextStateToQueue(Penalty, Node, /*NewLine=*/false);
-      addNextStateToQueue(Penalty, Node, /*NewLine=*/true);
+      FormatDecision LastFormat = Node->State.NextToken->Decision;
+      if (LastFormat == FD_Unformatted || LastFormat == FD_Continue)
+        addNextStateToQueue(Penalty, Node, /*NewLine=*/false);
+      if (LastFormat == FD_Unformatted || LastFormat == FD_Break)
+        addNextStateToQueue(Penalty, Node, /*NewLine=*/true);
     }
 
-    if (Queue.empty())
+    if (Queue.empty()) {
       // We were unable to find a solution, do nothing.
       // FIXME: Add diagnostic?
+      DEBUG(llvm::dbgs() << "Could not find a solution.\n");
       return 0;
+    }
 
     // Reconstruct the solution.
     if (!DryRun)
@@ -815,29 +820,54 @@ class Formatter : public UnwrappedLineConsumer {
             const std::vector<CharSourceRange> &Ranges)
       : Style(Style), Lex(Lex), SourceMgr(SourceMgr),
         Whitespaces(SourceMgr, Style, inputUsesCRLF(Lex.getBuffer())),
-        Ranges(Ranges), Encoding(encoding::detectEncoding(Lex.getBuffer())) {
+        Ranges(Ranges), UnwrappedLines(1),
+        Encoding(encoding::detectEncoding(Lex.getBuffer())) {
     DEBUG(llvm::dbgs() << "File encoding: "
                        << (Encoding == encoding::Encoding_UTF8 ? "UTF8"
                                                                : "unknown")
                        << "\n");
   }
 
-  virtual ~Formatter() {
-    for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) {
-      delete AnnotatedLines[i];
-    }
-  }
-
   tooling::Replacements format() {
+    tooling::Replacements Result;
     FormatTokenLexer Tokens(Lex, SourceMgr, Style, Encoding);
 
     UnwrappedLineParser Parser(Style, Tokens.lex(), *this);
     bool StructuralError = Parser.parse();
+    assert(UnwrappedLines.rbegin()->empty());
+    for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE;
+         ++Run) {
+      DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
+      SmallVector<AnnotatedLine *, 16> AnnotatedLines;
+      for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) {
+        AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i]));
+      }
+      tooling::Replacements RunResult =
+          format(AnnotatedLines, StructuralError, Tokens);
+      DEBUG({
+        llvm::dbgs() << "Replacements for run " << Run << ":\n";
+        for (tooling::Replacements::iterator I = RunResult.begin(),
+                                             E = RunResult.end();
+             I != E; ++I) {
+          llvm::dbgs() << I->toString() << "\n";
+        }
+      });
+      for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) {
+        delete AnnotatedLines[i];
+      }
+      Result.insert(RunResult.begin(), RunResult.end());
+      Whitespaces.reset();
+    }
+    return Result;
+  }
+
+  tooling::Replacements format(SmallVectorImpl<AnnotatedLine *> &AnnotatedLines,
+                               bool StructuralError, FormatTokenLexer &Tokens) {
     TokenAnnotator Annotator(Style, Tokens.getIdentTable().get("in"));
     for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) {
       Annotator.annotate(*AnnotatedLines[i]);
     }
-    deriveLocalStyle();
+    deriveLocalStyle(AnnotatedLines);
     for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) {
       Annotator.calculateFormattingInformation(*AnnotatedLines[i]);
     }
@@ -920,8 +950,7 @@ class Formatter : public UnwrappedLineConsumer {
       } else {
         // Format the first token if necessary, and notify the WhitespaceManager
         // about the unchanged whitespace.
-        for (const FormatToken *Tok = TheLine.First; Tok != NULL;
-             Tok = Tok->Next) {
+        for (FormatToken *Tok = TheLine.First; Tok != NULL; Tok = Tok->Next) {
           if (Tok == TheLine.First &&
               (Tok->NewlinesBefore > 0 || Tok->IsFirst)) {
             unsigned LevelIndent = Tok->OriginalColumn;
@@ -947,6 +976,9 @@ class Formatter : public UnwrappedLineConsumer {
         // last token.
         PreviousLineWasTouched = false;
       }
+      for (FormatToken *Tok = TheLine.First; Tok != NULL; Tok = Tok->Next) {
+        Tok->Finalized = true;
+      }
       PreviousLine = *I;
     }
     return Whitespaces.generateReplacements();
@@ -957,7 +989,8 @@ class Formatter : public UnwrappedLineConsumer {
     return Text.count('\r') * 2 > Text.count('\n');
   }
 
-  void deriveLocalStyle() {
+  void
+  deriveLocalStyle(const SmallVectorImpl<AnnotatedLine *> &AnnotatedLines) {
     unsigned CountBoundToVariable = 0;
     unsigned CountBoundToType = 0;
     bool HasCpp03IncompatibleFormat = false;
@@ -1229,13 +1262,18 @@ class Formatter : public UnwrappedLineConsumer {
   }
 
   virtual void consumeUnwrappedLine(const UnwrappedLine &TheLine) {
-    AnnotatedLines.push_back(new AnnotatedLine(TheLine));
+    assert(!UnwrappedLines.empty());
+    UnwrappedLines.back().push_back(TheLine);
+  }
+
+  virtual void finishRun() {
+    UnwrappedLines.push_back(SmallVector<UnwrappedLine, 16>());
   }
 
   /// \brief Add a new line and the required indent before the first Token
   /// of the \c UnwrappedLine if there was no structural parsing error.
   /// Returns the indent level of the \c UnwrappedLine.
-  void formatFirstToken(const FormatToken &RootToken,
+  void formatFirstToken(FormatToken &RootToken,
                         const AnnotatedLine *PreviousLine, unsigned Indent,
                         bool InPPDirective) {
     unsigned Newlines =
@@ -1272,7 +1310,7 @@ class Formatter : public UnwrappedLineConsumer {
   SourceManager &SourceMgr;
   WhitespaceManager Whitespaces;
   std::vector<CharSourceRange> Ranges;
-  SmallVector<AnnotatedLine *, 16> AnnotatedLines;
+  SmallVector<SmallVector<UnwrappedLine, 16>, 2> UnwrappedLines;
 
   encoding::Encoding Encoding;
   bool BinPackInconclusiveFunctions;

Diff for: lib/Format/FormatToken.cpp

@@ -145,6 +145,7 @@ void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
     bool HasRowWithSufficientColumns = false;
     unsigned Column = 0;
     for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
+      assert(i < MustBreakBeforeItem.size());
       if (MustBreakBeforeItem[i] || Column == Columns) {
         ++Format.LineCount;
         Column = 0;

Diff for: lib/Format/FormatToken.h

@@ -76,6 +76,12 @@ enum ParameterPackingKind {
   PPK_Inconclusive
 };
 
+enum FormatDecision {
+  FD_Unformatted,
+  FD_Continue,
+  FD_Break
+};
+
 class TokenRole;
 class AnnotatedLine;
 
@@ -93,7 +99,8 @@ struct FormatToken {
         LongestObjCSelectorName(0), FakeRParens(0),
         StartsBinaryExpression(false), EndsBinaryExpression(false),
         LastInChainOfCalls(false), PartOfMultiVariableDeclStmt(false),
-        MatchingParen(NULL), Previous(NULL), Next(NULL) {}
+        MatchingParen(NULL), Previous(NULL), Next(NULL),
+        Decision(FD_Unformatted), Finalized(false) {}
 
   /// \brief The \c Token.
   Token Tok;
@@ -338,6 +345,14 @@ struct FormatToken {
 
   SmallVector<AnnotatedLine *, 1> Children;
 
+  /// \brief Stores the formatting decision for the token once it was made.
+  FormatDecision Decision;
+
+  /// \brief If \c true, this token has been fully formatted (indented and
+  /// potentially re-formatted inside), and we do not allow further formatting
+  /// changes.
+  bool Finalized;
+
 private:
   // Disallow copying.
   FormatToken(const FormatToken &) LLVM_DELETED_FUNCTION;

Diff for: lib/Format/TokenAnnotator.cpp

@@ -523,11 +523,17 @@ class AnnotatingParser {
     if (CurrentToken != NULL)
       CurrentToken = CurrentToken->Next;
 
-    // Reset token type in case we have already looked at it and then recovered
-    // from an error (e.g. failure to find the matching >).
-    if (CurrentToken != NULL && CurrentToken->Type != TT_LambdaLSquare &&
-        CurrentToken->Type != TT_ImplicitStringLiteral)
-      CurrentToken->Type = TT_Unknown;
+    if (CurrentToken != NULL) {
+      // Reset token type in case we have already looked at it and then
+      // recovered from an error (e.g. failure to find the matching >).
+      if (CurrentToken->Type != TT_LambdaLSquare &&
+          CurrentToken->Type != TT_ImplicitStringLiteral)
+        CurrentToken->Type = TT_Unknown;
+      if (CurrentToken->Role)
+        CurrentToken->Role.reset(NULL);
+      CurrentToken->FakeLParens.clear();
+      CurrentToken->FakeRParens = 0;
+    }
   }
 
   /// \brief A struct to hold information valid in a specific context, e.g.
@@ -1482,10 +1488,13 @@ void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
                  << " C=" << Tok->CanBreakBefore << " T=" << Tok->Type
                  << " S=" << Tok->SpacesRequiredBefore
                  << " P=" << Tok->SplitPenalty << " Name=" << Tok->Tok.getName()
-                 << " PPK=" << Tok->PackingKind << " FakeLParens=";
+                 << " L=" << Tok->TotalLength << " PPK=" << Tok->PackingKind
+                 << " FakeLParens=";
     for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
       llvm::errs() << Tok->FakeLParens[i] << "/";
     llvm::errs() << " FakeRParens=" << Tok->FakeRParens << "\n";
+    if (Tok->Next == NULL)
+      assert(Tok == Line.Last);
     Tok = Tok->Next;
   }
   llvm::errs() << "----\n";