Skip to content

Commit 2ebe03a

Browse files
pruivoryanemerson
pruivo
authored and
ryanemerson
committed
Ensure cache configuration has correct number of owners
Closes #41558 Signed-off-by: Pedro Ruivo <[email protected]>
1 parent d7630b0 commit 2ebe03a

File tree

3 files changed

+38
-17
lines changed

3 files changed

+38
-17
lines changed

‎docs/guides/server/caching.adoc‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -170,19 +170,6 @@ apply the upper bound to. For example, to apply an upper-bound of `1000` to the
170170

171171
Setting a maximum cache size for `sessions`, `clientSessions`, `offlineSessions` and `offlineClientSessions` is not supported when volatile sessions are enabled.
172172

173-
=== Configuring caches for availability
174-
175-
Distributed caches replicate cache entries on a subset of nodes in a cluster and assigns entries to fixed owner nodes.
176-
177-
Each distributed cache, that is a primary source of truth of the data (`authenticationSessions`, `loginFailures` and `actionTokens`) has two owners per default, which means that two nodes have a copy of the specific cache entries.
178-
Non-owner nodes query the owners of a specific cache to obtain data.
179-
When one of the owners becomes unavailable, the data is restored from the remaining owner and rebalanced across the remaining nodes.
180-
When both owner nodes are offline, all data is lost.
181-
182-
The default number of two owners is the minimum number is necessary to survive one node (owner) failure or a rolling restart in a cluster setup with at least two nodes.
183-
A higher number increases the availability of the data, but at the expense of slower writes as more nodes need to be updated.
184-
Therefore, changing the number of owners for the caches `authenticationSessions`, `loginFailures` and `actionTokens` is not recommended.
185-
186173
=== Specify your own cache configuration file
187174

188175
To specify your own cache configuration file, enter this command:

‎model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/CacheConfigurator.java‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,20 @@
3434
import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup;
3535
import org.jboss.logging.Logger;
3636
import org.keycloak.Config;
37-
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
3837

38+
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ACTION_TOKEN_CACHE;
39+
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ALL_CACHES_NAME;
40+
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME;
3941
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME;
4042
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX;
4143
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME;
4244
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME;
45+
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES;
46+
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CRL_CACHE_DEFAULT_MAX;
4347
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CRL_CACHE_NAME;
4448
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOCAL_CACHE_NAMES;
4549
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOCAL_MAX_COUNT_CACHES;
50+
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME;
4651
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME;
4752
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME;
4853
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.REALM_CACHE_NAME;
@@ -102,7 +107,7 @@ public static void configureLocalCaches(Config.Scope keycloakConfig, Configurati
102107
*/
103108
public static void applyDefaultConfiguration(ConfigurationBuilderHolder holder) {
104109
var configs = holder.getNamedConfigurationBuilders();
105-
for (var name : InfinispanConnectionProvider.ALL_CACHES_NAME) {
110+
for (var name : ALL_CACHES_NAME) {
106111
configs.computeIfAbsent(name, cacheName -> DEFAULT_CONFIGS.getOrDefault(cacheName, TO_NULL).get());
107112
}
108113
}
@@ -157,7 +162,7 @@ public static void validateWorkCacheConfiguration(ConfigurationBuilderHolder hol
157162
*/
158163
public static void removeClusteredCaches(ConfigurationBuilderHolder holder) {
159164
logger.debug("Removing clustered caches");
160-
Arrays.stream(InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES).forEach(holder.getNamedConfigurationBuilders()::remove);
165+
Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(holder.getNamedConfigurationBuilders()::remove);
161166
}
162167

163168
/**
@@ -234,6 +239,34 @@ public static void configureSessionsCachesForVolatileSessions(ConfigurationBuild
234239
}
235240
}
236241

242+
/**
243+
* Configures the caches "actionToken", "authenticationSessions", and "loginFailures" with the minimum number of
244+
* owners to prevent data loss in a single instance crash.
245+
* <p>
246+
* The data in those caches only exist in memory, therefore they must have more than one owner configured.
247+
*
248+
* @param holder The {@link ConfigurationBuilderHolder} where the caches are configured.
249+
* @throws IllegalStateException if an Infinispan cache is not defined in the {@code holder}. This could indicate a
250+
* missing or incorrect configuration.
251+
*/
252+
public static void ensureMinimumOwners(ConfigurationBuilderHolder holder) {
253+
for (var name : Arrays.asList(
254+
LOGIN_FAILURE_CACHE_NAME,
255+
AUTHENTICATION_SESSIONS_CACHE_NAME,
256+
ACTION_TOKEN_CACHE)) {
257+
var builder = holder.getNamedConfigurationBuilders().get(name);
258+
if (builder == null) {
259+
throw cacheNotFound(name);
260+
}
261+
var hashConfig = builder.clustering().hash();
262+
var owners = hashConfig.attributes().attribute(HashConfiguration.NUM_OWNERS).get();
263+
if (owners < 2) {
264+
logger.infof("Setting num_owners=2 (configured value is %s) for cache '%s' to prevent data loss.", owners, name);
265+
hashConfig.numOwners(2);
266+
}
267+
}
268+
}
269+
237270
// private methods below
238271

239272
private static void configureRevisionCache(ConfigurationBuilderHolder holder, String baseCache, String revisionCache, long defaultMaxEntries) {
@@ -267,7 +300,7 @@ private static IllegalStateException cacheNotFound(String cache) {
267300

268301
public static ConfigurationBuilder getCrlCacheConfig() {
269302
var builder = createCacheConfigurationBuilder();
270-
builder.memory().whenFull(EvictionStrategy.REMOVE).maxCount(InfinispanConnectionProvider.CRL_CACHE_DEFAULT_MAX);
303+
builder.memory().whenFull(EvictionStrategy.REMOVE).maxCount(CRL_CACHE_DEFAULT_MAX);
271304
return builder;
272305
}
273306

‎model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/DefaultCacheEmbeddedConfigProviderFactory.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ private static void singleSiteConfiguration(Config.Scope config, ConfigurationBu
210210
CacheConfigurator.checkCachesExist(holder, Arrays.stream(ALL_CACHES_NAME));
211211
CacheConfigurator.configureCacheMaxCount(config, holder, Arrays.stream(CLUSTERED_MAX_COUNT_CACHES));
212212
CacheConfigurator.validateWorkCacheConfiguration(holder);
213+
CacheConfigurator.ensureMinimumOwners(holder);
213214
KeycloakModelUtils.runJobInTransaction(factory, session -> JGroupsConfigurator.configureJGroups(config, holder, session));
214215
configureMetrics(config, holder);
215216
}

0 commit comments

Comments
 (0)