[8.x] Do not look over TaskCancelledException when looking at failures when updating CCS info for clusters (#125206) (#125237)

* Do not look over `TaskCancelledException` when looking at failures when updating CCS info for clusters (#125206)

Do not look over `TaskCancelledException` when looking at failures when updating CCS info for clusters

(cherry picked from commit 0350a48)

# Conflicts:
#	muted-tests.yml

* Fix conflict

Author: pawankartik-elastic

muted-tests.yml

@@ -291,9 +291,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/117591
 - class: org.elasticsearch.repositories.s3.RepositoryS3ClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/117596
-- class: org.elasticsearch.search.ccs.CrossClusterIT
-  method: testCancel
-  issue: https://github.com/elastic/elasticsearch/issues/108061
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805

server/src/main/java/org/elasticsearch/ExceptionsHelper.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.NoSeedNodeLeftException;
 import org.elasticsearch.transport.NoSuchRemoteClusterException;
@@ -514,6 +515,15 @@ public static boolean isRemoteUnavailableException(Exception e) {
         return false;
     }
 
+    /**
+     * Utility method to check if an Exception is/was caused by TaskCancelledException.
+     * @param e Exception we're interested in evaluating.
+     * @return true if the Exception is/was caused by TaskCancelledException, else false.
+     */
+    public static boolean isTaskCancelledException(Exception e) {
+        return ExceptionsHelper.unwrapCausesAndSuppressed(e, ex -> ex instanceof TaskCancelledException).isPresent();
+    }
+
     private static class GroupBy {
         final String reason;
         final String index;

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -38,7 +38,6 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
-import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.transport.Transport;
 
 import java.util.ArrayList;
@@ -505,7 +504,7 @@ void onShardFailure(final int shardIndex, SearchShardTarget shardTarget, Excepti
         }
         // we don't aggregate shard on failures due to the internal cancellation,
         // but do keep the header counts right
-        if ((requestCancelled.get() && isTaskCancelledException(e)) == false) {
+        if ((requestCancelled.get() && ExceptionsHelper.isTaskCancelledException(e)) == false) {
             AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures.get();
             // lazily create shard failures, so we can early build the empty shard failure list in most cases (no failures)
             if (shardFailures == null) { // this is double checked locking but it's fine since SetOnce uses a volatile read internally
@@ -535,10 +534,6 @@ void onShardFailure(final int shardIndex, SearchShardTarget shardTarget, Excepti
         }
     }
 
-    private static boolean isTaskCancelledException(Exception e) {
-        return ExceptionsHelper.unwrapCausesAndSuppressed(e, ex -> ex instanceof TaskCancelledException).isPresent();
-    }
-
     /**
      * Executed once for every successful shard level request.
      * @param result the result returned form the shard

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -1709,7 +1709,7 @@ public final void onFailure(Exception e) {
             ShardSearchFailure f = new ShardSearchFailure(e);
             logCCSError(f, clusterAlias, skipUnavailable);
             SearchResponse.Cluster cluster = clusters.getCluster(clusterAlias);
-            if (skipUnavailable) {
+            if (skipUnavailable && ExceptionsHelper.isTaskCancelledException(e) == false) {
                 if (cluster != null) {
                     ccsClusterInfoUpdate(f, clusters, clusterAlias, true);
                 }
@@ -1718,7 +1718,8 @@ public final void onFailure(Exception e) {
                     ccsClusterInfoUpdate(f, clusters, clusterAlias, false);
                 }
                 Exception exception = e;
-                if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias) == false) {
+                if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias) == false
+                    && ExceptionsHelper.isTaskCancelledException(e) == false) {
                     exception = wrapRemoteClusterFailure(clusterAlias, e);
                 }
                 if (exceptions.compareAndSet(null, exception) == false) {