KEYCLOAK-5910 Retry sending notification when remote cache is unavailable

hmlnarik · hmlnarik · commit b466f4d0b6b1 · 2017-11-30T11:48:26.000+01:00
diff --git a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanNotificationsManager.java b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanNotificationsManager.java
@@ -50,8 +50,10 @@
 import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.common.util.ConcurrentMultivaluedHashMap;
+import org.keycloak.common.util.Retry;
 import org.keycloak.executors.ExecutorsProvider;
 import org.keycloak.models.KeycloakSession;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
 
 /**
  * Impl for sending infinispan messages across cluster and listening to them
@@ -153,7 +155,21 @@ void notify(String taskKey, ClusterEvent event, boolean ignoreSender, ClusterPro
                     .put(eventKey, wrappedEvent, 120, TimeUnit.SECONDS);
         } else {
             // Add directly to remoteCache. Will notify remote listeners on all nodes in all DCs
-            workRemoteCache.put(eventKey, wrappedEvent, 120, TimeUnit.SECONDS);
+            Retry.executeWithBackoff((int iteration) -> {
+                try {
+                    workRemoteCache.put(eventKey, wrappedEvent, 120, TimeUnit.SECONDS);
+                } catch (HotRodClientException re) {
+                if (logger.isDebugEnabled()) {
+                    logger.debugf(re, "Failed sending notification to remote cache '%s'. Key: '%s', iteration '%s'. Will try to retry the task",
+                            workRemoteCache.getName(), eventKey, iteration);
+                }
+
+                // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
+                throw re;
+            }
+
+        }, 10, 10);
+
         }
     }
 
