Make Jedis Cluster more likely to antirez's redis-rb-cluster

JedisClusterCommand

* improvements on connection error handling
** if based on slot connection throws connection related exception, retry to random node
** if we retry with random node, but all nodes are unreachable, throw JedisConnectionException without retry
** try to release connection whether connection is broken or not

* bug fix : if asking flag is on, and success this time, set asking flag to off

JedisClusterConnectionHandler

* have flexibility on initializing slots cache
** allow some nodes connection failure - skip
** if current node is success initializing slots cache, skip other nodes
** if current node failed to initialize slots cache, discard all discovered nodes and slots

* set nodes if node does not exist in nodes
** it restricts JedisPool to replace - prevent IllegalStateException : Returned object not currently part of this pool

JedisSlotBasedConnectionGuaranteedConnectionHandler

* getConnection (random connection)
** check all connections by random sequence
** always return valid connection (able to ping-pong)
** throw exception if all connections are invalid

* some refactoring

src/main/java/redis/clients/jedis/JedisClusterConnectionHandler.java

@@ -5,17 +5,22 @@ import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 
+import redis.clients.jedis.exceptions.JedisConnectionException;
+
 public abstract class JedisClusterConnectionHandler {
 
     protected Map<String, JedisPool> nodes = new HashMap<String, JedisPool>();
     protected Map<Integer, JedisPool> slots = new HashMap<Integer, JedisPool>();
 
     abstract Jedis getConnection();
-    
+
     protected void returnConnection(Jedis connection) {
-	nodes.get(
-		connection.getClient().getHost()
-			+ connection.getClient().getPort()).returnResource(
+	nodes.get(getNodeKey(connection.getClient()))
+		.returnResource(connection);
+    }
+
+    public void returnBrokenConnection(Jedis connection) {
+	nodes.get(getNodeKey(connection.getClient())).returnBrokenResource(
 		connection);
     }
 
@@ -29,29 +34,57 @@ public abstract class JedisClusterConnectionHandler {
 	return nodes;
     }
 
-    private void initializeSlotsCache(Set<HostAndPort> nodes) {
-	for (HostAndPort hostAndPort : nodes) {
+    private void initializeSlotsCache(Set<HostAndPort> startNodes) {
+	for (HostAndPort hostAndPort : startNodes) {
 	    JedisPool jp = new JedisPool(hostAndPort.getHost(),
 		    hostAndPort.getPort());
-	    this.nodes.put(hostAndPort.getHost() + hostAndPort.getPort(), jp);
-	    Jedis jedis = jp.getResource();
+
+	    this.nodes.clear();
+	    this.slots.clear();
+
+	    Jedis jedis = null;
 	    try {
+		jedis = jp.getResource();
 		discoverClusterNodesAndSlots(jedis);
+		break;
+	    } catch (JedisConnectionException e) {
+		if (jedis != null) {
+		    jp.returnBrokenResource(jedis);
+		    jedis = null;
+		}
+
+		// try next nodes
 	    } finally {
-		jp.returnResource(jedis);
+		if (jedis != null) {
+		    jp.returnResource(jedis);
+		}
 	    }
 	}
-    }
 
+	for (HostAndPort node : startNodes) {
+	    setNodeIfNotExist(node);
+	}
+    }
+    
     private void discoverClusterNodesAndSlots(Jedis jedis) {
 	String localNodes = jedis.clusterNodes();
 	for (String nodeInfo : localNodes.split("\n")) {
 	    HostAndPort node = getHostAndPortFromNodeLine(nodeInfo, jedis);
-	    JedisPool nodePool = new JedisPool(node.getHost(), node.getPort());
-	    this.nodes.put(node.getHost() + node.getPort(), nodePool);
+	    setNodeIfNotExist(node);
+	    
+	    JedisPool nodePool = nodes.get(getNodeKey(node));
 	    populateNodeSlots(nodeInfo, nodePool);
 	}
     }
+    
+    private void setNodeIfNotExist(HostAndPort node) {
+	String nodeKey = getNodeKey(node);
+	if (nodes.containsKey(nodeKey))
+	    return;
+	
+	JedisPool nodePool = new JedisPool(node.getHost(), node.getPort());
+	nodes.put(nodeKey, nodePool);
+    }
 
     private void populateNodeSlots(String nodeInfo, JedisPool nodePool) {
 	String[] nodeInfoArray = nodeInfo.split(" ");
@@ -74,7 +107,8 @@ public abstract class JedisClusterConnectionHandler {
 	}
     }
 
-    private HostAndPort getHostAndPortFromNodeLine(String nodeInfo, Jedis currentConnection) {
+    private HostAndPort getHostAndPortFromNodeLine(String nodeInfo,
+	    Jedis currentConnection) {
 	String stringHostAndPort = nodeInfo.split(" ", 3)[1];
 	if (":0".equals(stringHostAndPort)) {
 	    return new HostAndPort(currentConnection.getClient().getHost(),
@@ -86,9 +120,16 @@ public abstract class JedisClusterConnectionHandler {
     }
 
     public void assignSlotToNode(int slot, HostAndPort targetNode) {
-	JedisPool targetPool = nodes.get(targetNode.getHost()
-		+ targetNode.getPort());
-	slots.put(slot, targetPool);
+	JedisPool targetPool = nodes.get(getNodeKey(targetNode));
+
+	if (targetPool != null) {
+	    slots.put(slot, targetPool);
+	} else {
+	    setNodeIfNotExist(targetNode);
+	    
+	    targetPool = nodes.get(getNodeKey(targetNode));
+	    slots.put(slot, targetPool);
+	}
     }
 
     protected JedisPool getRandomConnection() {
@@ -96,4 +137,11 @@ public abstract class JedisClusterConnectionHandler {
 	return (JedisPool) (nodeArray[new Random().nextInt(nodeArray.length)]);
     }
 
+    protected String getNodeKey(HostAndPort hnp) {
+	return hnp.getHost() + ":" + hnp.getPort();
+    }
+
+    protected String getNodeKey(Client client) {
+	return client.getHost() + ":" + client.getPort();
+    }
 }