Avoid control flow for copying heap allocations to temporaries

Updates of all the temporaries that contain the address of a candidate
object for cold block heapification are currently guarded with ifacmpne
instructions.  In order to simplify the control flow graph produced for
heapification before cold blocks, replace the conditional update of each
temporary with an unconditional store using the result of an aternary
instruction that either updates the temporary with the heap object
address or with the current contents of the temporary unconditionally.
This update is inserted into the start of the cold block rather than in
a new block preceding the cold block.

This is only done if code generation supports ternary opcodes.  If
ternary opcodes are not supported, ifacmpne checks are still generated
instead.  The ifacmpne checks can also be forced by setting the
environment variable TR_disableTernaryOpForEA.

Signed-off-by:  Henry Zongaro <zongaro@ca.ibm.com>

Author: hzongaro

runtime/compiler/optimizer/EscapeAnalysis.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2019 IBM Corp. and others
+ * Copyright (c) 2000, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -1290,7 +1290,7 @@ int32_t TR_EscapeAnalysis::performAnalysisOnce()
 
             if (candidate->escapesInColdBlocks())
                {
-               heapifyBeforeColdBlocks(candidate);
+               heapifyForColdBlocks(candidate);
                if (candidate->_fields)
                   {
                   int32_t i;
@@ -7053,8 +7053,11 @@ void TR_EscapeAnalysis::makeNonContiguousLocalAllocation(Candidate *candidate)
 
 
 
-void TR_EscapeAnalysis::heapifyBeforeColdBlocks(Candidate *candidate)
+void TR_EscapeAnalysis::heapifyForColdBlocks(Candidate *candidate)
    {
+   static char *disableTernaryOpForEA = feGetEnv("TR_disableTernaryOpForEA");
+   bool useTernaryOp = !disableTernaryOpForEA && cg()->getSupportsTernary();
+
    if (comp()->suppressAllocationInlining())
       return;
 
@@ -7406,73 +7409,103 @@ void TR_EscapeAnalysis::heapifyBeforeColdBlocks(Candidate *candidate)
             }
         }
 
+      TR::TreeTop *insertSymRefStoresAfter = NULL;
+
+      // If using aternary to perform comparisons, all compares and stores are
+      // inserted directly at the start of the cold block
+      if (useTernaryOp)
+         {
+         insertSymRefStoresAfter = coldBlock->getEntry();
+         }
+
       ListIterator<TR::SymbolReference> symRefsIt(candidate->getSymRefs());
       TR::SymbolReference *symRef;
+      bool generatedReusedOperations = false;
+      TR::Node *heapTempLoad = NULL;
+      TR::Node *candidateStackAddrLoad = NULL;
+
       for (symRef = symRefsIt.getFirst(); symRef; symRef = symRefsIt.getNext())
         {
         //
-        // Now create the compares (one for each node) and
-        // stores if required
+        // Now create the compares (one for each node) and stores
         //
-        TR::Node *comparisonNode = TR::Node::createif(TR::ifacmpne, TR::Node::createWithSymRef(candidate->_node, TR::aload, 0, symRef), candidate->_node->duplicateTree(), targetBlock->getEntry());
-        TR::TreeTop *comparisonTree = TR::TreeTop::create(comp(), comparisonNode, NULL, NULL);
-        TR::Block *comparisonBlock = toBlock(cfg->addNode(TR::Block::createEmptyBlock(comparisonNode, comp(), coldBlock->getFrequency())));
-        comparisonBlock->inheritBlockInfo(coldBlock, coldBlock->isCold());
+        if (useTernaryOp)
+           {
+           // Reload address of object on heap just once for this block
+           if (!heapTempLoad)
+              {
+              heapTempLoad = TR::Node::createWithSymRef(candidate->_node, TR::aload, 0, heapSymRef);
+              candidateStackAddrLoad = candidate->_node->duplicateTree();
+              }
 
-        TR::TreeTop *comparisonEntryTree = comparisonBlock->getEntry();
-        TR::TreeTop *comparisonExitTree = comparisonBlock->getExit();
-        comparisonEntryTree->join(comparisonTree);
-        comparisonTree->join(comparisonExitTree);
+           // If variable has address of the stack allocated object, replace
+           // with the value of the heap allocated object; otherwise, keep the
+           // current value
+           //
+           // astore <object-temp>
+           //   aternary
+           //     acmpeq
+           //       aload <object-temp>
+           //       loadaddr <stack-obj>
+           //     aload <heap-allocated-obj>
+           //     aload <object-temp>
+           //
+           TR::Node *symLoad = TR::Node::createWithSymRef(candidate->_node, TR::aload, 0, symRef);
+           TR::Node *addrCompareNode = TR::Node::create(candidate->_node, TR::acmpeq, 2, symLoad, candidateStackAddrLoad);
+           TR::Node *chooseAddrNode = TR::Node::create(TR::aternary, 3, addrCompareNode, heapTempLoad, symLoad);
+
+           TR::TreeTop *storeTree = storeHeapifiedToTemp(candidate, chooseAddrNode, symRef);
+
+           storeTree->join(insertSymRefStoresAfter->getNextTreeTop());
+           insertSymRefStoresAfter->join(storeTree);
+           }
+        else
+           {
+           TR::Node *comparisonNode = TR::Node::createif(TR::ifacmpne, TR::Node::createWithSymRef(candidate->_node, TR::aload, 0, symRef), candidate->_node->duplicateTree(), targetBlock->getEntry());
+           TR::TreeTop *comparisonTree = TR::TreeTop::create(comp(), comparisonNode, NULL, NULL);
+           TR::Block *comparisonBlock = toBlock(cfg->addNode(TR::Block::createEmptyBlock(comparisonNode, comp(), coldBlock->getFrequency())));
+           comparisonBlock->inheritBlockInfo(coldBlock, coldBlock->isCold());
 
-        comparisonExitTree->join(insertionPoint);
+           TR::TreeTop *comparisonEntryTree = comparisonBlock->getEntry();
+           TR::TreeTop *comparisonExitTree = comparisonBlock->getExit();
+           comparisonEntryTree->join(comparisonTree);
+           comparisonTree->join(comparisonExitTree);
 
-        if (treeBeforeInsertionPoint)
-           treeBeforeInsertionPoint->join(comparisonEntryTree);
-        else
-           comp()->setStartTree(comparisonEntryTree);
+           comparisonExitTree->join(insertionPoint);
 
-        TR::Node *storeNode = TR::Node::createWithSymRef(TR::astore, 1, 1, TR::Node::createWithSymRef(comparisonNode, TR::aload, 0, heapSymRef), symRef);
-        if (symRef->getSymbol()->holdsMonitoredObject())
-           storeNode->setLiveMonitorInitStore(true);
-        storeNode->setHeapificationStore(true);
-        TR::TreeTop *storeTree = TR::TreeTop::create(comp(), storeNode, NULL, NULL);
+           if (treeBeforeInsertionPoint)
+              treeBeforeInsertionPoint->join(comparisonEntryTree);
+           else
+              comp()->setStartTree(comparisonEntryTree);
 
+           TR::Node *heapifiedObjAddrLoad = TR::Node::createWithSymRef(comparisonNode, TR::aload, 0, heapSymRef);
 
-        if (!symRef->getSymbol()->isParm())
-           {
-           TR::Node *initStoreNode = TR::Node::createWithSymRef(TR::astore, 1, 1, TR::Node::aconst(comparisonNode, 0), symRef);
-           if (symRef->getSymbol()->holdsMonitoredObject())
-              initStoreNode->setLiveMonitorInitStore(true);
-           TR::TreeTop *initStoreTree = TR::TreeTop::create(comp(), initStoreNode, NULL, NULL);
-           TR::TreeTop *startTree = comp()->getStartTree();
-           TR::TreeTop *nextToStart = startTree->getNextTreeTop();
-           startTree->join(initStoreTree);
-           initStoreTree->join(nextToStart);
-           }
+           TR::TreeTop *storeTree = storeHeapifiedToTemp(candidate, heapifiedObjAddrLoad, symRef);
 
-        TR::Block *storeBlock = toBlock(cfg->addNode(TR::Block::createEmptyBlock(storeNode, comp(), coldBlock->getFrequency())));
-        storeBlock->inheritBlockInfo(coldBlock, coldBlock->isCold());
+           TR::Block *storeBlock = toBlock(cfg->addNode(TR::Block::createEmptyBlock(storeTree->getNode(), comp(), coldBlock->getFrequency())));
+           storeBlock->inheritBlockInfo(coldBlock, coldBlock->isCold());
 
-        cfg->addEdge(comparisonBlock, storeBlock);
-        cfg->addEdge(comparisonBlock, targetBlock);
-        cfg->addEdge(storeBlock, targetBlock);
-        if (targetBlock == coldBlock)
-          {
-          lastComparisonBlock = comparisonBlock;
-          lastStoreBlock = storeBlock;
-          }
+           cfg->addEdge(comparisonBlock, storeBlock);
+           cfg->addEdge(comparisonBlock, targetBlock);
+           cfg->addEdge(storeBlock, targetBlock);
+           if (targetBlock == coldBlock)
+             {
+             lastComparisonBlock = comparisonBlock;
+             lastStoreBlock = storeBlock;
+             }
 
-        TR::TreeTop *storeEntryTree = storeBlock->getEntry();
-        TR::TreeTop *storeExitTree = storeBlock->getExit();
+           TR::TreeTop *storeEntryTree = storeBlock->getEntry();
+           TR::TreeTop *storeExitTree = storeBlock->getExit();
 
-        comparisonExitTree->join(storeEntryTree);
-        storeEntryTree->join(storeTree);
-        storeTree->join(storeExitTree);
-        storeExitTree->join(insertionPoint);
+           comparisonExitTree->join(storeEntryTree);
+           storeEntryTree->join(storeTree);
+           storeTree->join(storeExitTree);
+           storeExitTree->join(insertionPoint);
 
-        insertionPoint = comparisonEntryTree;
-        treeBeforeInsertionPoint = insertionPoint->getPrevTreeTop();
-        targetBlock = comparisonBlock;
+           insertionPoint = comparisonEntryTree;
+           treeBeforeInsertionPoint = insertionPoint->getPrevTreeTop();
+           targetBlock = comparisonBlock;
+           }
         }
 
       cfg->addEdge(heapAllocationBlock, targetBlock);
@@ -7484,9 +7517,11 @@ void TR_EscapeAnalysis::heapifyBeforeColdBlocks(Candidate *candidate)
          TR::CFGNode *predNode = (*pred)->getFrom();
          /* might be removed, keep reference to next object in list */
          pred++;
-         if (((predNode != lastComparisonBlock) &&
-              (predNode != lastStoreBlock)) ||
-             coldBlock->isCatchBlock())
+         if ((useTernaryOp && (predNode != heapComparisonBlock)
+                 && (predNode != heapAllocationBlock))
+             || (!useTernaryOp && (predNode != lastComparisonBlock)
+                 && (predNode != lastStoreBlock))
+             || coldBlock->isCatchBlock())
             {
             TR::Block *predBlock = toBlock(predNode);
             if (!coldBlock->isCatchBlock() &&
@@ -7607,6 +7642,31 @@ void TR_EscapeAnalysis::heapifyBeforeColdBlocks(Candidate *candidate)
    }
 
 
+TR::TreeTop *TR_EscapeAnalysis::storeHeapifiedToTemp(Candidate *candidate, TR::Node *value, TR::SymbolReference *symRef)
+   {
+   TR::Node *storeNode = TR::Node::createWithSymRef(TR::astore, 1, 1, value, symRef);
+   TR::TreeTop *storeTree = TR::TreeTop::create(comp(), storeNode, NULL, NULL);
+
+   if (symRef->getSymbol()->holdsMonitoredObject())
+      {
+      storeNode->setLiveMonitorInitStore(true);
+      }
+   storeNode->setHeapificationStore(true);
+
+   if (!symRef->getSymbol()->isParm())
+      {
+      TR::Node *initStoreNode = TR::Node::createWithSymRef(TR::astore, 1, 1, TR::Node::aconst(candidate->_node, 0), symRef);
+      if (symRef->getSymbol()->holdsMonitoredObject())
+         initStoreNode->setLiveMonitorInitStore(true);
+      TR::TreeTop *initStoreTree = TR::TreeTop::create(comp(), initStoreNode, NULL, NULL);
+      TR::TreeTop *startTree = comp()->getStartTree();
+      TR::TreeTop *nextToStart = startTree->getNextTreeTop();
+           startTree->join(initStoreTree);
+      initStoreTree->join(nextToStart);
+      }
+
+   return storeTree;
+   }
 
 
 bool TR_EscapeAnalysis::devirtualizeCallSites()

runtime/compiler/optimizer/EscapeAnalysis.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2019 IBM Corp. and others
+ * Copyright (c) 2000, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -590,7 +590,23 @@ class TR_EscapeAnalysis : public TR::Optimization
 
    void     makeContiguousLocalAllocation(Candidate *candidate);
    void     makeNonContiguousLocalAllocation(Candidate *candidate);
-   void     heapifyBeforeColdBlocks(Candidate *candidate);
+   void     heapifyForColdBlocks(Candidate *candidate);
+
+   /**
+    * \brief Store the supplied address to the specified temporary
+    *
+    * \param candidate
+    *     The candidate that is being heapified
+    *
+    * \param addr
+    *     The address of the possibly heapified object
+    *
+    * \param symRef
+    *     The \ref TR::SymbolReference for the temporay
+    *
+    * \return A pointer to the \ref TR::TreeTop containing the store
+    */
+   TR::TreeTop *storeHeapifiedToTemp(Candidate *candidate, TR::Node *addr, TR::SymbolReference *symRef);
    bool     inlineCallSites();
    void     scanForExtraCallsToInline();
    bool     alwaysWorthInlining(TR::Node *callNode);