rik/jlibredis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use "sentinel failover" to force failover

Browse Source 
* Use "sentinel failover" to force failover
** faster than kill redis instances
* set failover timeout to 1 min
** It makes sense with failover within localhost
* reduce instances : 1 Redis Server and 1 Sentinel
** port is not changed -> I'll changed later at end of refactoring
This commit is contained in:
Jungtaek Lim
2014-02-15 10:30:01 +09:00
parent b2fa6b6c40
commit d2e073b2f2
2 changed files with 17 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
Makefile
View File

@@ -25,6 +25,7 @@ define REDIS3_CONF
daemonize yes
port 6381
requirepass foobared
masterauth foobared
pidfile /tmp/redis3.pid
logfile /tmp/redis3.log
save ""
@@ -43,18 +44,6 @@ appendonly no
slaveof localhost 6381
endef
define REDIS5_CONF
daemonize yes
port 6383
requirepass foobared
masterauth foobared
pidfile /tmp/redis5.pid
logfile /tmp/redis5.log
save ""
appendonly no
slaveof localhost 6381
endef
define REDIS6_CONF
daemonize yes
port 6384
@@ -97,7 +86,7 @@ daemonize yes
sentinel monitor mymaster 127.0.0.1 6379 1
sentinel auth-pass mymaster foobared
sentinel down-after-milliseconds mymaster 2000
sentinel failover-timeout mymaster 180000
sentinel failover-timeout mymaster 60000
sentinel parallel-syncs mymaster 1
pidfile /tmp/sentinel1.pid
logfile /tmp/sentinel1.log
@@ -106,34 +95,22 @@ endef
define REDIS_SENTINEL2
port 26380
daemonize yes
sentinel monitor mymaster 127.0.0.1 6381 2
sentinel monitor mymaster 127.0.0.1 6381 1
sentinel auth-pass mymaster foobared
sentinel down-after-milliseconds mymaster 2000
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 180000
sentinel failover-timeout mymaster 60000
pidfile /tmp/sentinel2.pid
logfile /tmp/sentinel2.log
endef
define REDIS_SENTINEL3
port 26381
daemonize yes
sentinel monitor mymaster 127.0.0.1 6381 2
sentinel auth-pass mymaster foobared
sentinel down-after-milliseconds mymaster 2000
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 180000
pidfile /tmp/sentinel3.pid
logfile /tmp/sentinel3.log
endef
define REDIS_SENTINEL4
port 26382
daemonize yes
sentinel monitor mymasterfailover 127.0.0.1 6385 1
sentinel auth-pass mymasterfailover foobared
sentinel down-after-milliseconds mymasterfailover 2000
sentinel failover-timeout mymasterfailover 180000
sentinel failover-timeout mymasterfailover 60000
sentinel parallel-syncs mymasterfailover 1
pidfile /tmp/sentinel4.pid
logfile /tmp/sentinel4.log
@@ -180,13 +157,11 @@ export REDIS1_CONF
export REDIS2_CONF
export REDIS3_CONF
export REDIS4_CONF
export REDIS5_CONF
export REDIS6_CONF
export REDIS7_CONF
export REDIS8_CONF
export REDIS_SENTINEL1
export REDIS_SENTINEL2
export REDIS_SENTINEL3
export REDIS_SENTINEL4
export REDIS_CLUSTER_NODE1_CONF
export REDIS_CLUSTER_NODE2_CONF
@@ -197,7 +172,6 @@ start: cleanup
echo "$$REDIS2_CONF" | redis-server -
echo "$$REDIS3_CONF" | redis-server -
echo "$$REDIS4_CONF" | redis-server -
echo "$$REDIS5_CONF" | redis-server -
echo "$$REDIS6_CONF" | redis-server -
echo "$$REDIS7_CONF" | redis-server -
echo "$$REDIS8_CONF" | redis-server -
@@ -205,8 +179,6 @@ start: cleanup
@sleep 0.5
echo "$$REDIS_SENTINEL2" > /tmp/sentinel2.conf && redis-server /tmp/sentinel2.conf --sentinel
@sleep 0.5
echo "$$REDIS_SENTINEL3" > /tmp/sentinel3.conf && redis-server /tmp/sentinel3.conf --sentinel
@sleep 0.5
echo "$$REDIS_SENTINEL4" > /tmp/sentinel4.conf && redis-server /tmp/sentinel4.conf --sentinel
echo "$$REDIS_CLUSTER_NODE1_CONF" | redis-server -
echo "$$REDIS_CLUSTER_NODE2_CONF" | redis-server -
@@ -219,23 +191,19 @@ cleanup:
stop:
kill `cat /tmp/redis1.pid`
kill `cat /tmp/redis2.pid`
# this get's segfaulted by the tests
kill `cat /tmp/redis3.pid` || true
kill `cat /tmp/redis4.pid` || true
kill `cat /tmp/redis5.pid` || true
kill `cat /tmp/redis6.pid` || true
kill `cat /tmp/redis3.pid`
kill `cat /tmp/redis4.pid`
kill `cat /tmp/redis6.pid`
kill `cat /tmp/redis7.pid`
kill `cat /tmp/redis8.pid`
kill `cat /tmp/sentinel1.pid`
kill `cat /tmp/sentinel2.pid`
kill `cat /tmp/sentinel3.pid`
kill `cat /tmp/sentinel4.pid`
kill `cat /tmp/redis_cluster_node1.pid` || true
kill `cat /tmp/redis_cluster_node2.pid` || true
kill `cat /tmp/redis_cluster_node3.pid` || true
rm -f /tmp/sentinel1.conf
rm -f /tmp/sentinel2.conf
rm -f /tmp/sentinel3.conf
rm -f /tmp/sentinel4.conf
rm -f /tmp/redis_cluster_node1.conf
rm -f /tmp/redis_cluster_node2.conf

25
src/test/java/redis/clients/jedis/tests/JedisSentinelPoolTest.java
View File

@@ -14,6 +14,7 @@ import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPubSub;
import redis.clients.jedis.JedisSentinelPool;
import redis.clients.jedis.Transaction;
import redis.clients.jedis.exceptions.JedisDataException;
public class JedisSentinelPoolTest extends JedisTestBase {
private static final String MASTER_NAME = "mymaster";
@@ -26,8 +27,6 @@ public class JedisSentinelPoolTest extends JedisTestBase {
.get(4);
protected static HostAndPort sentinel1 = HostAndPortUtil
.getSentinelServers().get(1);
protected static HostAndPort sentinel2 = HostAndPortUtil
.getSentinelServers().get(2);
protected static Jedis sentinelJedis1;
@@ -36,7 +35,6 @@ public class JedisSentinelPoolTest extends JedisTestBase {
@Before
public void setUp() throws Exception {
sentinels.add(sentinel1.toString());
sentinels.add(sentinel2.toString());
sentinelJedis1 = new Jedis(sentinel1.getHost(), sentinel1.getPort());
}
@@ -46,30 +44,25 @@ public class JedisSentinelPoolTest extends JedisTestBase {
JedisSentinelPool pool = new JedisSentinelPool(MASTER_NAME, sentinels,
new GenericObjectPoolConfig(), 1000, "foobared", 2);
// perform failover
doSegFaultMaster(pool);
// perform failover once again
doSegFaultMaster(pool);
forceFailover(pool);
forceFailover(pool);
// you can test failover as much as possible
// but you need to prepare additional slave per failover
}
private void doSegFaultMaster(JedisSentinelPool pool)
throws InterruptedException {
private void forceFailover(JedisSentinelPool pool) throws InterruptedException {
HostAndPort oldMaster = pool.getCurrentHostMaster();
// jedis connection should be master
Jedis jedis = pool.getResource();
assertEquals("PONG", jedis.ping());
try {
jedis.debug(DebugParams.SEGFAULT());
} catch (Exception e) {
}
// It can throw JedisDataException while there's no slave to promote
// There's nothing we can do, so we just pass Exception to make test fail fast
sentinelJedis1.sentinelFailover(MASTER_NAME);
waitForFailover(pool, oldMaster);
// JedisSentinelPool recognize master but may not changed internal pool yet
Thread.sleep(100);
jedis = pool.getResource();